HADOOP-6794. Move configuration and script files post split. Includes HDFS-1181, MAPREDUCE-1033.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@953490 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Thomas White
parent
f07af58599
commit
32469f1bdf
|
|
@ -935,6 +935,8 @@ Release 0.21.0 - Unreleased
|
|||
HADOOP-6738. Move cluster_setup.xml from MapReduce to Common.
|
||||
(Tom White via tomwhite)
|
||||
|
||||
HADOOP-6794. Move configuration and script files post split. (tomwhite)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
HADOOP-5595. NameNode does not need to run a replicator to choose a
|
||||
|
|
|
|||
|
|
@ -53,7 +53,9 @@ case $COMMAND in
|
|||
echo ""
|
||||
#try to locate hdfs and if present, delegate to it.
|
||||
if [ -f "${HADOOP_HDFS_HOME}"/bin/hdfs ]; then
|
||||
exec "${HADOOP_HDFS_HOME}"/bin/hdfs $*
|
||||
exec "${HADOOP_HDFS_HOME}"/bin/hdfs $*
|
||||
elif [ -f "${HADOOP_HOME}"/bin/hdfs ]; then
|
||||
exec "${HADOOP_HOME}"/bin/hdfs $*
|
||||
else
|
||||
echo "HDFS not found."
|
||||
exit
|
||||
|
|
@ -67,7 +69,9 @@ case $COMMAND in
|
|||
echo ""
|
||||
#try to locate mapred and if present, delegate to it.
|
||||
if [ -f "${HADOOP_MAPRED_HOME}"/bin/mapred ]; then
|
||||
exec "${HADOOP_MAPRED_HOME}"/bin/mapred $*
|
||||
exec "${HADOOP_MAPRED_HOME}"/bin/mapred $*
|
||||
elif [ -f "${HADOOP_HOME}"/bin/mapred ]; then
|
||||
exec "${HADOOP_HOME}"/bin/mapred $*
|
||||
else
|
||||
echo "MAPRED not found."
|
||||
exit
|
||||
|
|
|
|||
|
|
@ -31,17 +31,17 @@ while [ -h "$this" ]; do
|
|||
done
|
||||
|
||||
# convert relative path to absolute path
|
||||
bin=`dirname "$this"`
|
||||
common_bin=`dirname "$this"`
|
||||
script=`basename "$this"`
|
||||
bin=`cd "$bin"; pwd`
|
||||
this="$bin/$script"
|
||||
common_bin=`cd "$common_bin"; pwd`
|
||||
this="$common_bin/$script"
|
||||
|
||||
# the root of the Hadoop installation
|
||||
#TODO: change the env variable when dir structure is changed
|
||||
export HADOOP_HOME=`dirname "$this"`/..
|
||||
export HADOOP_CORE_HOME="${HADOOP_HOME}"
|
||||
export HADOOP_COMMON_HOME="${HADOOP_HOME}"
|
||||
#export HADOOP_HOME=`dirname "$this"`/../..
|
||||
#export HADOOP_CORE_HOME="${HADOOP_CORE_HOME:-`dirname "$this"`/..}"
|
||||
#export HADOOP_COMMON_HOME="${HADOOP_COMMON_HOME:-`dirname "$this"`/..}"
|
||||
|
||||
#check to see if the conf dir is given as an optional argument
|
||||
if [ $# -gt 1 ]
|
||||
|
|
@ -106,54 +106,54 @@ CLASSPATH="${HADOOP_CONF_DIR}"
|
|||
CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar
|
||||
|
||||
# for developers, add Hadoop classes to CLASSPATH
|
||||
if [ -d "$HADOOP_CORE_HOME/build/classes" ]; then
|
||||
CLASSPATH=${CLASSPATH}:$HADOOP_CORE_HOME/build/classes
|
||||
if [ -d "$HADOOP_COMMON_HOME/build/classes" ]; then
|
||||
CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/build/classes
|
||||
fi
|
||||
if [ -d "$HADOOP_CORE_HOME/build/webapps" ]; then
|
||||
CLASSPATH=${CLASSPATH}:$HADOOP_CORE_HOME/build
|
||||
if [ -d "$HADOOP_COMMON_HOME/build/webapps" ]; then
|
||||
CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/build
|
||||
fi
|
||||
if [ -d "$HADOOP_CORE_HOME/build/test/classes" ]; then
|
||||
CLASSPATH=${CLASSPATH}:$HADOOP_CORE_HOME/build/test/classes
|
||||
if [ -d "$HADOOP_COMMON_HOME/build/test/classes" ]; then
|
||||
CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/build/test/classes
|
||||
fi
|
||||
if [ -d "$HADOOP_CORE_HOME/build/test/core/classes" ]; then
|
||||
CLASSPATH=${CLASSPATH}:$HADOOP_CORE_HOME/build/test/core/classes
|
||||
if [ -d "$HADOOP_COMMON_HOME/build/test/core/classes" ]; then
|
||||
CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/build/test/core/classes
|
||||
fi
|
||||
|
||||
# so that filenames w/ spaces are handled correctly in loops below
|
||||
IFS=
|
||||
|
||||
# for releases, add core hadoop jar & webapps to CLASSPATH
|
||||
if [ -d "$HADOOP_CORE_HOME/webapps" ]; then
|
||||
CLASSPATH=${CLASSPATH}:$HADOOP_CORE_HOME
|
||||
if [ -d "$HADOOP_COMMON_HOME/webapps" ]; then
|
||||
CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME
|
||||
fi
|
||||
for f in $HADOOP_CORE_HOME/hadoop-core-*.jar; do
|
||||
for f in $HADOOP_COMMON_HOME/hadoop-*.jar; do
|
||||
CLASSPATH=${CLASSPATH}:$f;
|
||||
done
|
||||
|
||||
# add libs to CLASSPATH
|
||||
for f in $HADOOP_CORE_HOME/lib/*.jar; do
|
||||
for f in $HADOOP_COMMON_HOME/lib/*.jar; do
|
||||
CLASSPATH=${CLASSPATH}:$f;
|
||||
done
|
||||
|
||||
if [ -d "$HADOOP_CORE_HOME/build/ivy/lib/Hadoop-Core/common" ]; then
|
||||
for f in $HADOOP_CORE_HOME/build/ivy/lib/Hadoop-Core/common/*.jar; do
|
||||
if [ -d "$HADOOP_COMMON_HOME/build/ivy/lib/Hadoop-Common/common" ]; then
|
||||
for f in $HADOOP_COMMON_HOME/build/ivy/lib/Hadoop-Common/common/*.jar; do
|
||||
CLASSPATH=${CLASSPATH}:$f;
|
||||
done
|
||||
fi
|
||||
|
||||
if [ -d "$HADOOP_CORE_HOME/build/ivy/lib/Hadoop-Hdfs/common" ]; then
|
||||
for f in $HADOOP_CORE_HOME/build/ivy/lib/Hadoop-Hdfs/common/*.jar; do
|
||||
if [ -d "$HADOOP_COMMON_HOME/build/ivy/lib/Hadoop-Hdfs/common" ]; then
|
||||
for f in $HADOOP_COMMON_HOME/build/ivy/lib/Hadoop-Hdfs/common/*.jar; do
|
||||
CLASSPATH=${CLASSPATH}:$f;
|
||||
done
|
||||
fi
|
||||
|
||||
if [ -d "$HADOOP_CORE_HOME/build/ivy/lib/Hadoop/common" ]; then
|
||||
for f in $HADOOP_CORE_HOME/build/ivy/lib/Hadoop/common/*.jar; do
|
||||
if [ -d "$HADOOP_COMMON_HOME/build/ivy/lib/Hadoop/common" ]; then
|
||||
for f in $HADOOP_COMMON_HOME/build/ivy/lib/Hadoop/common/*.jar; do
|
||||
CLASSPATH=${CLASSPATH}:$f;
|
||||
done
|
||||
fi
|
||||
|
||||
for f in $HADOOP_CORE_HOME/lib/jsp-2.1/*.jar; do
|
||||
for f in $HADOOP_COMMON_HOME/lib/jsp-2.1/*.jar; do
|
||||
CLASSPATH=${CLASSPATH}:$f;
|
||||
done
|
||||
|
||||
|
|
@ -180,23 +180,23 @@ unset IFS
|
|||
|
||||
# cygwin path translation
|
||||
if $cygwin; then
|
||||
HADOOP_CORE_HOME=`cygpath -w "$HADOOP_CORE_HOME"`
|
||||
HADOOP_COMMON_HOME=`cygpath -w "$HADOOP_COMMON_HOME"`
|
||||
HADOOP_LOG_DIR=`cygpath -w "$HADOOP_LOG_DIR"`
|
||||
fi
|
||||
# setup 'java.library.path' for native-hadoop code if necessary
|
||||
JAVA_LIBRARY_PATH=''
|
||||
if [ -d "${HADOOP_CORE_HOME}/build/native" -o -d "${HADOOP_CORE_HOME}/lib/native" ]; then
|
||||
if [ -d "${HADOOP_COMMON_HOME}/build/native" -o -d "${HADOOP_COMMON_HOME}/lib/native" ]; then
|
||||
JAVA_PLATFORM=`CLASSPATH=${CLASSPATH} ${JAVA} -Xmx32m ${HADOOP_JAVA_PLATFORM_OPTS} org.apache.hadoop.util.PlatformName | sed -e "s/ /_/g"`
|
||||
|
||||
if [ -d "$HADOOP_CORE_HOME/build/native" ]; then
|
||||
JAVA_LIBRARY_PATH=${HADOOP_CORE_HOME}/build/native/${JAVA_PLATFORM}/lib
|
||||
if [ -d "$HADOOP_COMMON_HOME/build/native" ]; then
|
||||
JAVA_LIBRARY_PATH=${HADOOP_COMMON_HOME}/build/native/${JAVA_PLATFORM}/lib
|
||||
fi
|
||||
|
||||
if [ -d "${HADOOP_CORE_HOME}/lib/native" ]; then
|
||||
if [ -d "${HADOOP_COMMON_HOME}/lib/native" ]; then
|
||||
if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
|
||||
JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:${HADOOP_CORE_HOME}/lib/native/${JAVA_PLATFORM}
|
||||
JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:${HADOOP_COMMON_HOME}/lib/native/${JAVA_PLATFORM}
|
||||
else
|
||||
JAVA_LIBRARY_PATH=${HADOOP_CORE_HOME}/lib/native/${JAVA_PLATFORM}
|
||||
JAVA_LIBRARY_PATH=${HADOOP_COMMON_HOME}/lib/native/${JAVA_PLATFORM}
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
|
@ -208,7 +208,7 @@ fi
|
|||
|
||||
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR"
|
||||
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.file=$HADOOP_LOGFILE"
|
||||
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.home.dir=$HADOOP_CORE_HOME"
|
||||
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.home.dir=$HADOOP_COMMON_HOME"
|
||||
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.id.str=$HADOOP_IDENT_STRING"
|
||||
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.root.logger=${HADOOP_ROOT_LOGGER:-INFO,console}"
|
||||
if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
|
||||
|
|
@ -225,6 +225,15 @@ if [ "$HADOOP_HDFS_HOME" = "" ]; then
|
|||
fi
|
||||
|
||||
if [ -d "${HADOOP_HDFS_HOME}" ]; then
|
||||
|
||||
if [ -d "$HADOOP_HDFS_HOME/webapps" ]; then
|
||||
CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME
|
||||
fi
|
||||
|
||||
if [ -d "${HADOOP_HDFS_HOME}/conf" ]; then
|
||||
CLASSPATH=${CLASSPATH}:${HADOOP_HDFS_HOME}/conf
|
||||
fi
|
||||
|
||||
for f in $HADOOP_HDFS_HOME/hadoop-hdfs-*.jar; do
|
||||
CLASSPATH=${CLASSPATH}:$f;
|
||||
done
|
||||
|
|
@ -253,6 +262,15 @@ if [ "$HADOOP_MAPRED_HOME" = "" ]; then
|
|||
fi
|
||||
|
||||
if [ -d "${HADOOP_MAPRED_HOME}" ]; then
|
||||
|
||||
if [ -d "$HADOOP_MAPRED_HOME/webapps" ]; then
|
||||
CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME
|
||||
fi
|
||||
|
||||
if [ -d "${HADOOP_MAPRED_HOME}/conf" ]; then
|
||||
CLASSPATH=${CLASSPATH}:${HADOOP_MAPRED_HOME}/conf
|
||||
fi
|
||||
|
||||
for f in $HADOOP_MAPRED_HOME/hadoop-mapred-*.jar; do
|
||||
CLASSPATH=${CLASSPATH}:$f
|
||||
done
|
||||
|
|
@ -284,6 +302,3 @@ if $cygwin; then
|
|||
fi
|
||||
|
||||
|
||||
# TODO:remove this when dir structure is changed
|
||||
#export HADOOP_HDFS_HOME=$HADOOP_HOME
|
||||
#export HADOOP_MAPRED_HOME=$HADOOP_HOME
|
||||
|
|
|
|||
113
bin/hdfs
113
bin/hdfs
|
|
@ -1,113 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
bin=`dirname "$0"`
|
||||
bin=`cd "$bin"; pwd`
|
||||
|
||||
. "$bin"/hdfs-config.sh
|
||||
|
||||
function print_usage(){
|
||||
echo "Usage: hdfs [--config confdir] COMMAND"
|
||||
echo " where COMMAND is one of:"
|
||||
echo " namenode -format format the DFS filesystem"
|
||||
echo " secondarynamenode run the DFS secondary namenode"
|
||||
echo " namenode run the DFS namenode"
|
||||
echo " datanode run a DFS datanode"
|
||||
echo " dfsadmin run a DFS admin client"
|
||||
echo " fsck run a DFS filesystem checking utility"
|
||||
echo " balancer run a cluster balancing utility"
|
||||
echo " jmxget get JMX exported values from NameNode or DataNode."
|
||||
echo " oiv apply the offline fsimage viewer to an fsimage"
|
||||
echo " fetchdt fetch a delegation token from the NameNode"
|
||||
echo " Use -help to see options"
|
||||
echo ""
|
||||
echo "Most commands print help when invoked w/o parameters."
|
||||
}
|
||||
|
||||
if [ $# = 0 ]; then
|
||||
print_usage
|
||||
exit
|
||||
fi
|
||||
|
||||
COMMAND=$1
|
||||
shift
|
||||
|
||||
if [ "$COMMAND" = "namenode" ] ; then
|
||||
CLASS='org.apache.hadoop.hdfs.server.namenode.NameNode'
|
||||
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_NAMENODE_OPTS"
|
||||
elif [ "$COMMAND" = "secondarynamenode" ] ; then
|
||||
CLASS='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode'
|
||||
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_SECONDARYNAMENODE_OPTS"
|
||||
elif [ "$COMMAND" = "datanode" ] ; then
|
||||
CLASS='org.apache.hadoop.hdfs.server.datanode.DataNode'
|
||||
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_DATANODE_OPTS"
|
||||
elif [ "$COMMAND" = "dfs" ] ; then
|
||||
CLASS=org.apache.hadoop.fs.FsShell
|
||||
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
|
||||
elif [ "$COMMAND" = "dfsadmin" ] ; then
|
||||
CLASS=org.apache.hadoop.hdfs.tools.DFSAdmin
|
||||
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
|
||||
elif [ "$COMMAND" = "fsck" ] ; then
|
||||
CLASS=org.apache.hadoop.hdfs.tools.DFSck
|
||||
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
|
||||
elif [ "$COMMAND" = "balancer" ] ; then
|
||||
CLASS=org.apache.hadoop.hdfs.server.balancer.Balancer
|
||||
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_BALANCER_OPTS"
|
||||
elif [ "$COMMAND" = "jmxget" ] ; then
|
||||
CLASS=org.apache.hadoop.hdfs.tools.JMXGet
|
||||
elif [ "$COMMAND" = "oiv" ] ; then
|
||||
CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer
|
||||
elif [ "$COMMAND" = "fetchdt" ] ; then
|
||||
CLASS=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher
|
||||
else
|
||||
echo $COMMAND - invalid command
|
||||
print_usage
|
||||
exit
|
||||
fi
|
||||
|
||||
# for developers, add hdfs classes to CLASSPATH
|
||||
if [ -d "$HADOOP_HDFS_HOME/build/classes" ]; then
|
||||
CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/build/classes
|
||||
fi
|
||||
if [ -d "$HADOOP_HDFS_HOME/build/webapps" ]; then
|
||||
CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/build
|
||||
fi
|
||||
if [ -d "$HADOOP_HDFS_HOME/build/test/classes" ]; then
|
||||
CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/build/test/classes
|
||||
fi
|
||||
if [ -d "$HADOOP_HDFS_HOME/build/tools" ]; then
|
||||
CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/build/tools
|
||||
fi
|
||||
|
||||
# for releases, add core hdfs jar & webapps to CLASSPATH
|
||||
if [ -d "$HADOOP_HDFS_HOME/webapps" ]; then
|
||||
CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME
|
||||
fi
|
||||
for f in $HADOOP_HDFS_HOME/hadoop-hdfs-*.jar; do
|
||||
CLASSPATH=${CLASSPATH}:$f;
|
||||
done
|
||||
|
||||
# add libs to CLASSPATH
|
||||
for f in $HADOOP_HDFS_HOME/lib/*.jar; do
|
||||
CLASSPATH=${CLASSPATH}:$f;
|
||||
done
|
||||
|
||||
if $cygwin; then
|
||||
CLASSPATH=`cygpath -p -w "$CLASSPATH"`
|
||||
fi
|
||||
export CLASSPATH=$CLASSPATH
|
||||
exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS $CLASS "$@"
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# included in all the hdfs scripts with source command
|
||||
# should not be executed directly
|
||||
|
||||
bin=`dirname "$0"`
|
||||
bin=`cd "$bin"; pwd`
|
||||
|
||||
#TODO: change the env variable when directory structure is changed
|
||||
export HADOOP_CORE_HOME="${HADOOP_CORE_HOME:-$bin/..}"
|
||||
#export HADOOP_CORE_HOME="${HADOOP_CORE_HOME:-$bin/../../core}"
|
||||
|
||||
if [ -d "${HADOOP_CORE_HOME}" ]; then
|
||||
. "$HADOOP_CORE_HOME"/bin/hadoop-config.sh
|
||||
else
|
||||
echo "Hadoop core not found."
|
||||
exit
|
||||
fi
|
||||
100
bin/mapred
100
bin/mapred
|
|
@ -1,100 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
bin=`dirname "$0"`
|
||||
bin=`cd "$bin"; pwd`
|
||||
|
||||
. $bin/mapred-config.sh
|
||||
|
||||
function print_usage(){
|
||||
echo "Usage: mapred [--config confdir] COMMAND"
|
||||
echo " where COMMAND is one of:"
|
||||
echo " mradmin run a Map-Reduce admin client"
|
||||
echo " jobtracker run the MapReduce job Tracker node"
|
||||
echo " tasktracker run a MapReduce task Tracker node"
|
||||
echo " pipes run a Pipes job"
|
||||
echo " job manipulate MapReduce jobs"
|
||||
echo " queue get information regarding JobQueues"
|
||||
echo ""
|
||||
echo "Most commands print help when invoked w/o parameters."
|
||||
}
|
||||
|
||||
if [ $# = 0 ]; then
|
||||
print_usage
|
||||
exit
|
||||
fi
|
||||
|
||||
COMMAND=$1
|
||||
shift
|
||||
|
||||
if [ "$COMMAND" = "mradmin" ] ; then
|
||||
CLASS=org.apache.hadoop.mapred.tools.MRAdmin
|
||||
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
|
||||
elif [ "$COMMAND" = "jobtracker" ] ; then
|
||||
CLASS=org.apache.hadoop.mapred.JobTracker
|
||||
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_JOBTRACKER_OPTS"
|
||||
elif [ "$COMMAND" = "tasktracker" ] ; then
|
||||
CLASS=org.apache.hadoop.mapred.TaskTracker
|
||||
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_TASKTRACKER_OPTS"
|
||||
elif [ "$COMMAND" = "job" ] ; then
|
||||
CLASS=org.apache.hadoop.mapred.JobClient
|
||||
elif [ "$COMMAND" = "queue" ] ; then
|
||||
CLASS=org.apache.hadoop.mapred.JobQueueClient
|
||||
elif [ "$COMMAND" = "pipes" ] ; then
|
||||
CLASS=org.apache.hadoop.mapred.pipes.Submitter
|
||||
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
|
||||
elif [ "$COMMAND" = "sampler" ] ; then
|
||||
CLASS=org.apache.hadoop.mapred.lib.InputSampler
|
||||
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
|
||||
else
|
||||
echo $COMMAND - invalid command
|
||||
print_usage
|
||||
exit
|
||||
fi
|
||||
|
||||
# for developers, add mapred classes to CLASSPATH
|
||||
if [ -d "$HADOOP_MAPRED_HOME/build/classes" ]; then
|
||||
CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build/classes
|
||||
fi
|
||||
if [ -d "$HADOOP_MAPRED_HOME/build/webapps" ]; then
|
||||
CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build
|
||||
fi
|
||||
if [ -d "$HADOOP_MAPRED_HOME/build/test/classes" ]; then
|
||||
CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build/test/classes
|
||||
fi
|
||||
if [ -d "$HADOOP_MAPRED_HOME/build/tools" ]; then
|
||||
CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build/tools
|
||||
fi
|
||||
|
||||
# for releases, add core mapred jar & webapps to CLASSPATH
|
||||
if [ -d "$HADOOP_MAPRED_HOME/webapps" ]; then
|
||||
CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME
|
||||
fi
|
||||
for f in $HADOOP_MAPRED_HOME/hadoop-mapred-*.jar; do
|
||||
CLASSPATH=${CLASSPATH}:$f;
|
||||
done
|
||||
|
||||
# add libs to CLASSPATH
|
||||
for f in $HADOOP_MAPRED_HOME/lib/*.jar; do
|
||||
CLASSPATH=${CLASSPATH}:$f;
|
||||
done
|
||||
|
||||
if $cygwin; then
|
||||
CLASSPATH=`cygpath -p -w "$CLASSPATH"`
|
||||
fi
|
||||
export CLASSPATH
|
||||
exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS $CLASS "$@"
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# included in all the mapred scripts with source command
|
||||
# should not be executed directly
|
||||
|
||||
bin=`dirname "$0"`
|
||||
bin=`cd "$bin"; pwd`
|
||||
|
||||
#TODO: change the env variable when directory structure is changed
|
||||
export HADOOP_CORE_HOME="${HADOOP_CORE_HOME:-$bin/..}"
|
||||
#export HADOOP_CORE_HOME="${HADOOP_CORE_HOME:-$bin/../../core}"
|
||||
|
||||
if [ -d "${HADOOP_CORE_HOME}" ]; then
|
||||
. "$HADOOP_CORE_HOME"/bin/hadoop-config.sh
|
||||
else
|
||||
echo "Hadoop core not found."
|
||||
exit
|
||||
fi
|
||||
2
bin/rcc
2
bin/rcc
|
|
@ -72,7 +72,7 @@ IFS=
|
|||
if [ -d "$HADOOP_HOME/webapps" ]; then
|
||||
CLASSPATH=${CLASSPATH}:$HADOOP_HOME
|
||||
fi
|
||||
for f in $HADOOP_HOME/hadoop-core-*.jar; do
|
||||
for f in $HADOOP_HOME/hadoop-*.jar; do
|
||||
CLASSPATH=${CLASSPATH}:$f;
|
||||
done
|
||||
|
||||
|
|
|
|||
|
|
@ -1,25 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
bin=`dirname "${BASH_SOURCE-$0}"`
|
||||
bin=`cd "$bin"; pwd`
|
||||
|
||||
. "$bin"/hdfs-config.sh
|
||||
|
||||
# Start balancer daemon.
|
||||
|
||||
"$HADOOP_CORE_HOME"/bin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs start balancer $@
|
||||
|
|
@ -1,52 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
# Start hadoop dfs daemons.
|
||||
# Optinally upgrade or rollback dfs state.
|
||||
# Run this on master node.
|
||||
|
||||
usage="Usage: start-dfs.sh [-upgrade|-rollback]"
|
||||
|
||||
bin=`dirname "${BASH_SOURCE-$0}"`
|
||||
bin=`cd "$bin"; pwd`
|
||||
|
||||
. "$bin"/hdfs-config.sh
|
||||
|
||||
# get arguments
|
||||
if [ $# -ge 1 ]; then
|
||||
nameStartOpt=$1
|
||||
shift
|
||||
case $nameStartOpt in
|
||||
(-upgrade)
|
||||
;;
|
||||
(-rollback)
|
||||
dataStartOpt=$nameStartOpt
|
||||
;;
|
||||
(*)
|
||||
echo $usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# start dfs daemons
|
||||
# start namenode after datanodes, to minimize time namenode is up w/o data
|
||||
# note: datanodes will log connection errors until namenode starts
|
||||
"$HADOOP_CORE_HOME"/bin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs start namenode $nameStartOpt
|
||||
"$HADOOP_CORE_HOME"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs start datanode $dataStartOpt
|
||||
"$HADOOP_CORE_HOME"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --hosts masters --script "$bin"/hdfs start secondarynamenode
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
# Start hadoop map reduce daemons. Run this on master node.
|
||||
|
||||
bin=`dirname "${BASH_SOURCE-$0}"`
|
||||
bin=`cd "$bin"; pwd`
|
||||
|
||||
. $bin/mapred-config.sh
|
||||
|
||||
# start mapred daemons
|
||||
# start jobtracker first to minimize connection errors at startup
|
||||
"$HADOOP_CORE_HOME"/bin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/mapred start jobtracker
|
||||
"$HADOOP_CORE_HOME"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/mapred start tasktracker
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
bin=`dirname "${BASH_SOURCE-$0}"`
|
||||
bin=`cd "$bin"; pwd`
|
||||
|
||||
. "$bin"/hdfs-config.sh
|
||||
|
||||
# Stop balancer daemon.
|
||||
# Run this on the machine where the balancer is running
|
||||
|
||||
"$HADOOP_CORE_HOME"/bin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs stop balancer
|
||||
|
|
@ -1,28 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
# Stop hadoop DFS daemons. Run this on master node.
|
||||
|
||||
bin=`dirname "${BASH_SOURCE-$0}"`
|
||||
bin=`cd "$bin"; pwd`
|
||||
|
||||
. "$bin"/hdfs-config.sh
|
||||
|
||||
"$HADOOP_CORE_HOME"/bin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs stop namenode
|
||||
"$HADOOP_CORE_HOME"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs stop datanode
|
||||
"$HADOOP_CORE_HOME"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --hosts masters --script "$bin"/hdfs stop secondarynamenode
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
# Stop hadoop map reduce daemons. Run this on master node.
|
||||
|
||||
bin=`dirname "${BASH_SOURCE-$0}"`
|
||||
bin=`cd "$bin"; pwd`
|
||||
|
||||
. $bin/mapred-config.sh
|
||||
|
||||
"$HADOOP_CORE_HOME"/bin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/mapred stop jobtracker
|
||||
"$HADOOP_CORE_HOME"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/mapred stop tasktracker
|
||||
|
|
@ -1,126 +0,0 @@
|
|||
<?xml version="1.0"?>
|
||||
|
||||
<!-- This is the configuration file for the resource manager in Hadoop. -->
|
||||
<!-- You can configure various scheduling parameters related to queues. -->
|
||||
<!-- The properties for a queue follow a naming convention,such as, -->
|
||||
<!-- mapred.capacity-scheduler.queue.<queue-name>.property-name. -->
|
||||
|
||||
<configuration>
|
||||
|
||||
<property>
|
||||
<name>mapred.capacity-scheduler.queue.default.capacity</name>
|
||||
<value>100</value>
|
||||
<description>Percentage of the number of slots in the cluster that are
|
||||
to be available for jobs in this queue.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>mapred.capacity-scheduler.queue.default.supports-priority</name>
|
||||
<value>false</value>
|
||||
<description>If true, priorities of jobs will be taken into
|
||||
account in scheduling decisions.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>mapred.capacity-scheduler.queue.default.minimum-user-limit-percent</name>
|
||||
<value>100</value>
|
||||
<description> Each queue enforces a limit on the percentage of resources
|
||||
allocated to a user at any given time, if there is competition for them.
|
||||
This user limit can vary between a minimum and maximum value. The former
|
||||
depends on the number of users who have submitted jobs, and the latter is
|
||||
set to this property value. For example, suppose the value of this
|
||||
property is 25. If two users have submitted jobs to a queue, no single
|
||||
user can use more than 50% of the queue resources. If a third user submits
|
||||
a job, no single user can use more than 33% of the queue resources. With 4
|
||||
or more users, no user can use more than 25% of the queue's resources. A
|
||||
value of 100 implies no user limits are imposed.
|
||||
</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>mapred.capacity-scheduler.queue.default.maximum-initialized-jobs-per-user</name>
|
||||
<value>2</value>
|
||||
<description>The maximum number of jobs to be pre-initialized for a user
|
||||
of the job queue.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<!-- The default configuration settings for the capacity task scheduler -->
|
||||
<!-- The default values would be applied to all the queues which don't have -->
|
||||
<!-- the appropriate property for the particular queue -->
|
||||
<property>
|
||||
<name>mapred.capacity-scheduler.default-supports-priority</name>
|
||||
<value>false</value>
|
||||
<description>If true, priorities of jobs will be taken into
|
||||
account in scheduling decisions by default in a job queue.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>mapred.capacity-scheduler.task.default-pmem-percentage-in-vmem</name>
|
||||
<value>-1</value>
|
||||
<description>A percentage (float) of the default VM limit for jobs
|
||||
(mapred.task.default.maxvm). This is the default RAM task-limit
|
||||
associated with a task. Unless overridden by a job's setting, this
|
||||
number defines the RAM task-limit.
|
||||
|
||||
If this property is missing, or set to an invalid value, scheduling
|
||||
based on physical memory, RAM, is disabled.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>mapred.capacity-scheduler.task.limit.maxpmem</name>
|
||||
<value>-1</value>
|
||||
<description>Configuration that provides an upper limit on the maximum
|
||||
physical memory that can be specified by a job. The job configuration
|
||||
mapred.task.maxpmem should be less than this value. If not, the job will
|
||||
be rejected by the scheduler.
|
||||
|
||||
If it is set to -1, scheduler will not consider physical memory for
|
||||
scheduling even if virtual memory based scheduling is enabled(by setting
|
||||
valid values for both mapred.task.default.maxvmem and
|
||||
mapred.task.limit.maxvmem).
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>mapred.capacity-scheduler.default-minimum-user-limit-percent</name>
|
||||
<value>100</value>
|
||||
<description>The percentage of the resources limited to a particular user
|
||||
for the job queue at any given point of time by default.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>mapred.capacity-scheduler.default-maximum-initialized-jobs-per-user</name>
|
||||
<value>2</value>
|
||||
<description>The maximum number of jobs to be pre-initialized for a user
|
||||
of the job queue.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
|
||||
<!-- Capacity scheduler Job Initialization configuration parameters -->
|
||||
<property>
|
||||
<name>mapred.capacity-scheduler.init-poll-interval</name>
|
||||
<value>5000</value>
|
||||
<description>The amount of time in miliseconds which is used to poll
|
||||
the job queues for jobs to initialize.
|
||||
</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>mapred.capacity-scheduler.init-worker-threads</name>
|
||||
<value>5</value>
|
||||
<description>Number of worker threads which would be used by
|
||||
Initialization poller to initialize jobs in a set of queue.
|
||||
If number mentioned in property is equal to number of job queues
|
||||
then a single thread would initialize jobs in a queue. If lesser
|
||||
then a thread would get a set of queues assigned. If the number
|
||||
is greater then number of threads would be equal to number of
|
||||
job queues.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
</configuration>
|
||||
|
|
@ -94,4 +94,13 @@
|
|||
A special value of "*" means all users are allowed.</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>security.admin.operations.protocol.acl</name>
|
||||
<value>*</value>
|
||||
<description>ACL for AdminOperationsProtocol, used by the mradmins commands
|
||||
to refresh queues and nodes at JobTracker. The ACL is a comma-separated list of
|
||||
user and group names. The user and group list is separated by a blank.
|
||||
For e.g. "alice,bob users,wheel". A special value of "*" means all users are
|
||||
allowed.</description>
|
||||
</property>
|
||||
</configuration>
|
||||
|
|
|
|||
|
|
@ -1,8 +0,0 @@
|
|||
<?xml version="1.0"?>
|
||||
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
|
||||
|
||||
<!-- Put site-specific property overrides in this file. -->
|
||||
|
||||
<configuration>
|
||||
|
||||
</configuration>
|
||||
|
|
@ -3,6 +3,16 @@ hadoop.root.logger=INFO,console
|
|||
hadoop.log.dir=.
|
||||
hadoop.log.file=hadoop.log
|
||||
|
||||
#
|
||||
# Job Summary Appender
|
||||
#
|
||||
# Use following logger to send summary to separate file defined by
|
||||
# hadoop.mapreduce.jobsummary.log.file rolled daily:
|
||||
# hadoop.mapreduce.jobsummary.logger=INFO,JSA
|
||||
#
|
||||
hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger}
|
||||
hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log
|
||||
|
||||
# Define the root logger to the system property "hadoop.root.logger".
|
||||
log4j.rootLogger=${hadoop.root.logger}, EventCounter
|
||||
|
||||
|
|
@ -45,6 +55,7 @@ log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}:
|
|||
|
||||
#Default values
|
||||
hadoop.tasklog.taskid=null
|
||||
hadoop.tasklog.iscleanup=false
|
||||
hadoop.tasklog.noKeepSplits=4
|
||||
hadoop.tasklog.totalLogFileSize=100
|
||||
hadoop.tasklog.purgeLogSplits=true
|
||||
|
|
@ -52,12 +63,12 @@ hadoop.tasklog.logsRetainHours=12
|
|||
|
||||
log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender
|
||||
log4j.appender.TLA.taskId=${hadoop.tasklog.taskid}
|
||||
log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}
|
||||
log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}
|
||||
|
||||
log4j.appender.TLA.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
|
||||
|
||||
|
||||
#
|
||||
#Security appender
|
||||
#
|
||||
|
|
@ -105,3 +116,34 @@ log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR
|
|||
# Sends counts of logging messages at different severity levels to Hadoop Metrics.
|
||||
#
|
||||
log4j.appender.EventCounter=org.apache.hadoop.metrics.jvm.EventCounter
|
||||
|
||||
#
|
||||
# Job Summary Appender
|
||||
#
|
||||
log4j.appender.JSA=org.apache.log4j.DailyRollingFileAppender
|
||||
log4j.appender.JSA.File=${hadoop.log.dir}/${hadoop.mapreduce.jobsummary.log.file}
|
||||
log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
|
||||
log4j.appender.JSA.DatePattern=.yyyy-MM-dd
|
||||
log4j.logger.org.apache.hadoop.mapred.JobInProgress$JobSummary=${hadoop.mapreduce.jobsummary.logger}
|
||||
log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false
|
||||
|
||||
#
|
||||
# MapReduce Audit Log Appender
|
||||
#
|
||||
|
||||
# Set the MapReduce audit log filename
|
||||
#hadoop.mapreduce.audit.log.file=hadoop-mapreduce.audit.log
|
||||
|
||||
# Appender for AuditLogger.
|
||||
# Requires the following system properties to be set
|
||||
# - hadoop.log.dir (Hadoop Log directory)
|
||||
# - hadoop.mapreduce.audit.log.file (MapReduce audit log filename)
|
||||
|
||||
#log4j.logger.org.apache.hadoop.mapred.AuditLogger=INFO,MRAUDIT
|
||||
#log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false
|
||||
#log4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender
|
||||
#log4j.appender.MRAUDIT.File=${hadoop.log.dir}/${hadoop.mapreduce.audit.log.file}
|
||||
#log4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd
|
||||
#log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout
|
||||
#log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
|
||||
|
|
|
|||
|
|
@ -1,31 +0,0 @@
|
|||
<?xml version="1.0"?>
|
||||
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
|
||||
|
||||
<!-- This is a template file for queue acls configuration properties -->
|
||||
|
||||
<configuration>
|
||||
|
||||
<property>
|
||||
<name>mapred.queue.default.acl-submit-job</name>
|
||||
<value>*</value>
|
||||
<description> Comma separated list of user and group names that are allowed
|
||||
to submit jobs to the 'default' queue. The user list and the group list
|
||||
are separated by a blank. For e.g. alice,bob group1,group2.
|
||||
If set to the special value '*', it means all users are allowed to
|
||||
submit jobs.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>mapred.queue.default.acl-administer-jobs</name>
|
||||
<value>*</value>
|
||||
<description> Comma separated list of user and group names that are allowed
|
||||
to delete jobs or modify job's priority for jobs not owned by the current
|
||||
user in the 'default' queue. The user list and the group list
|
||||
are separated by a blank. For e.g. alice,bob group1,group2.
|
||||
If set to the special value '*', it means all users are allowed to do
|
||||
this operation.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
</configuration>
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
<?xml version="1.0"?>
|
||||
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
|
||||
|
||||
<!-- Put site-specific property overrides in this file. -->
|
||||
|
||||
<configuration>
|
||||
|
||||
</configuration>
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
mapred.local.dir=#configured value of hadoop.tmp.dir it can be a list of paths comma seperated
|
||||
hadoop.pid.dir=#configured HADOOP_PID_DIR
|
||||
hadoop.indent.str=#configured HADOOP_IDENT_STR
|
||||
Loading…
Reference in New Issue