diff --git a/dev-support/smart-apply-patch.sh b/dev-support/smart-apply-patch.sh index 3334c2bd882..c5386b4a2bb 100755 --- a/dev-support/smart-apply-patch.sh +++ b/dev-support/smart-apply-patch.sh @@ -53,16 +53,22 @@ if ! grep -qv '^a/\|^b/' $TMP ; then sed -i -e 's,^[ab]/,,' $TMP fi -# if all of the lines start with common/, hdfs/, or mapreduce/, this is +PREFIX_DIRS=$(cut -d '/' -f 1 $TMP | sort | uniq) + +# if we are at the project root then nothing more to do +if [[ -d hadoop-common ]]; then + echo Looks like this is being run at project root + +# if all of the lines start with hadoop-common/, hdfs/, or mapreduce/, this is # relative to the hadoop root instead of the subproject root, so we need # to chop off another layer -PREFIX_DIRS=$(cut -d '/' -f 1 $TMP | sort | uniq) -if [[ "$PREFIX_DIRS" =~ ^(hdfs|common|mapreduce)$ ]]; then +elif [[ "$PREFIX_DIRS" =~ ^(hdfs|hadoop-common|mapreduce)$ ]]; then echo Looks like this is relative to project root. Increasing PLEVEL PLEVEL=$[$PLEVEL + 1] -elif ! echo "$PREFIX_DIRS" | grep -vxq 'common\|hdfs\|mapreduce' ; then - echo Looks like this is a cross-subproject patch. Not supported! + +elif ! echo "$PREFIX_DIRS" | grep -vxq 'hadoop-common\|hdfs\|mapreduce' ; then + echo Looks like this is a cross-subproject patch. Try applying from the project root exit 1 fi diff --git a/dev-support/test-patch.properties b/dev-support/test-patch.properties index e459e3c9e17..2f1a994ceb6 100644 --- a/dev-support/test-patch.properties +++ b/dev-support/test-patch.properties @@ -13,6 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# The number of acceptable warning for *all* modules +# Please update the per-module test-patch.properties if you update this file. + OK_RELEASEAUDIT_WARNINGS=0 OK_FINDBUGS_WARNINGS=0 -OK_JAVADOC_WARNINGS=0 +OK_JAVADOC_WARNINGS=6 diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh index c5ba1c3e826..536bee8cd3e 100755 --- a/dev-support/test-patch.sh +++ b/dev-support/test-patch.sh @@ -19,84 +19,152 @@ ulimit -n 1024 ### SVN_REVISION and BUILD_URL are set by Hudson if it is run by patch process ### Read variables from properties file bindir=$(dirname $0) -. $bindir/test-patch.properties + +# Defaults +if [ -z "$MAVEN_HOME" ]; then + MVN=mvn +else + MVN=$MAVEN_HOME/bin/mvn +fi + +PROJECT_NAME=Hadoop +JENKINS=false +PATCH_DIR=/tmp +SUPPORT_DIR=/tmp +BASEDIR=$(pwd) + +PS=${PS:-ps} +AWK=${AWK:-awk} +WGET=${WGET:-wget} +SVN=${SVN:-svn} +GREP=${GREP:-grep} +PATCH=${PATCH:-patch} +JIRACLI=${JIRA:-jira} +FINDBUGS_HOME=${FINDBUGS_HOME} +FORREST_HOME=${FORREST_HOME} +ECLIPSE_HOME=${ECLIPSE_HOME} + +############################################################################### +printUsage() { + echo "Usage: $0 [options] patch-file | defect-number" + echo + echo "Where:" + echo " patch-file is a local patch file containing the changes to test" + echo " defect-number is a JIRA defect number (e.g. 'HADOOP-1234') to test (Jenkins only)" + echo + echo "Options:" + echo "--patch-dir= The directory for working and output files (default '/tmp')" + echo "--basedir= The directory to apply the patch to (default current directory)" + echo "--mvn-cmd= The 'mvn' command to use (default \$MAVEN_HOME/bin/mvn, or 'mvn')" + echo "--ps-cmd= The 'ps' command to use (default 'ps')" + echo "--awk-cmd= The 'awk' command to use (default 'awk')" + echo "--svn-cmd= The 'svn' command to use (default 'svn')" + echo "--grep-cmd= The 'grep' command to use (default 'grep')" + echo "--patch-cmd= The 'patch' command to use (default 'patch')" + echo "--findbugs-home= Findbugs home directory (default FINDBUGS_HOME environment variable)" + echo "--forrest-home= Forrest home directory (default FORREST_HOME environment variable)" + echo "--dirty-workspace Allow the local SVN workspace to have uncommitted changes" + echo + echo "Jenkins-only options:" + echo "--jenkins Run by Jenkins (runs tests and posts results to JIRA)" + echo "--support-dir= The directory to find support files in" + echo "--wget-cmd= The 'wget' command to use (default 'wget')" + echo "--jira-cmd= The 'jira' command to use (default 'jira')" + echo "--jira-password= The password for the 'jira' command" + echo "--eclipse-home= Eclipse home directory (default ECLIPSE_HOME environment variable)" +} ############################################################################### parseArgs() { - case "$1" in - HUDSON) - ### Set HUDSON to true to indicate that this script is being run by Hudson - HUDSON=true - if [[ $# != 16 ]] ; then - echo "ERROR: usage $0 HUDSON " - cleanupAndExit 0 - fi - PATCH_DIR=$2 - SUPPORT_DIR=$3 - PS=$4 - WGET=$5 - JIRACLI=$6 - SVN=$7 - GREP=$8 - PATCH=$9 - FINDBUGS_HOME=${10} - FORREST_HOME=${11} - ECLIPSE_HOME=${12} - BASEDIR=${13} - JIRA_PASSWD=${14} - CURL=${15} - defect=${16} - - ### Retrieve the defect number - if [ -z "$defect" ] ; then - echo "Could not determine the patch to test. Exiting." - cleanupAndExit 0 - fi - - if [ ! -e "$PATCH_DIR" ] ; then - mkdir -p $PATCH_DIR - fi - - ECLIPSE_PROPERTY="-Declipse.home=$ECLIPSE_HOME" + for i in $* + do + case $i in + --jenkins) + JENKINS=true ;; - DEVELOPER) - ### Set HUDSON to false to indicate that this script is being run by a developer - HUDSON=false - if [[ $# != 9 ]] ; then - echo "ERROR: usage $0 DEVELOPER " - cleanupAndExit 0 - fi - ### PATCH_FILE contains the location of the patchfile - PATCH_FILE=$2 - if [[ ! -e "$PATCH_FILE" ]] ; then - echo "Unable to locate the patch file $PATCH_FILE" - cleanupAndExit 0 - fi - PATCH_DIR=$3 - ### Check if $PATCH_DIR exists. If it does not exist, create a new directory - if [[ ! -e "$PATCH_DIR" ]] ; then - mkdir "$PATCH_DIR" - if [[ $? == 0 ]] ; then - echo "$PATCH_DIR has been created" - else - echo "Unable to create $PATCH_DIR" - cleanupAndExit 0 - fi - fi - SVN=$4 - GREP=$5 - PATCH=$6 - FINDBUGS_HOME=$7 - FORREST_HOME=$8 - BASEDIR=$9 - ### Obtain the patch filename to append it to the version number - defect=`basename $PATCH_FILE` + --patch-dir=*) + PATCH_DIR=${i#*=} + ;; + --support-dir=*) + SUPPORT_DIR=${i#*=} + ;; + --basedir=*) + BASEDIR=${i#*=} + ;; + --mvn-cmd=*) + MVN=${i#*=} + ;; + --ps-cmd=*) + PS=${i#*=} + ;; + --awk-cmd=*) + AWK=${i#*=} + ;; + --wget-cmd=*) + WGET=${i#*=} + ;; + --svn-cmd=*) + SVN=${i#*=} + ;; + --grep-cmd=*) + GREP=${i#*=} + ;; + --patch-cmd=*) + PATCH=${i#*=} + ;; + --jira-cmd=*) + JIRACLI=${i#*=} + ;; + --jira-password=*) + JIRA_PASSWD=${i#*=} + ;; + --findbugs-home=*) + FINDBUGS_HOME=${i#*=} + ;; + --forrest-home=*) + FORREST_HOME=${i#*=} + ;; + --eclipse-home=*) + ECLIPSE_HOME=${i#*=} + ;; + --dirty-workspace) + DIRTY_WORKSPACE=true ;; *) - echo "ERROR: usage $0 HUDSON [args] | DEVELOPER [args]" - cleanupAndExit 0 + PATCH_OR_DEFECT=$i ;; - esac + esac + done + if [ -z "$PATCH_OR_DEFECT" ]; then + printUsage + exit 1 + fi + if [[ $JENKINS == "true" ]] ; then + echo "Running in Jenkins mode" + defect=$PATCH_OR_DEFECT + ECLIPSE_PROPERTY="-Declipse.home=$ECLIPSE_HOME" + else + echo "Running in developer mode" + JENKINS=false + ### PATCH_FILE contains the location of the patchfile + PATCH_FILE=$PATCH_OR_DEFECT + if [[ ! -e "$PATCH_FILE" ]] ; then + echo "Unable to locate the patch file $PATCH_FILE" + cleanupAndExit 0 + fi + ### Check if $PATCH_DIR exists. If it does not exist, create a new directory + if [[ ! -e "$PATCH_DIR" ]] ; then + mkdir "$PATCH_DIR" + if [[ $? == 0 ]] ; then + echo "$PATCH_DIR has been created" + else + echo "Unable to create $PATCH_DIR" + cleanupAndExit 0 + fi + fi + ### Obtain the patch filename to append it to the version number + defect=`basename $PATCH_FILE` + fi } ############################################################################### @@ -111,9 +179,10 @@ checkout () { echo "" echo "" ### When run by a developer, if the workspace contains modifications, do not continue + ### unless the --dirty-workspace option was set status=`$SVN stat --ignore-externals | sed -e '/^X[ ]*/D'` - if [[ $HUDSON == "false" ]] ; then - if [[ "$status" != "" ]] ; then + if [[ $JENKINS == "false" ]] ; then + if [[ "$status" != "" && -z $DIRTY_WORKSPACE ]] ; then echo "ERROR: can't run in a workspace that contains the following modifications" echo "$status" cleanupAndExit 1 @@ -131,7 +200,7 @@ checkout () { ############################################################################### setup () { ### Download latest patch file (ignoring .htm and .html) when run from patch process - if [[ $HUDSON == "true" ]] ; then + if [[ $JENKINS == "true" ]] ; then $WGET -q -O $PATCH_DIR/jira http://issues.apache.org/jira/browse/$defect if [[ `$GREP -c 'Patch Available' $PATCH_DIR/jira` == 0 ]] ; then echo "$defect is not \"Patch Available\". Exiting." @@ -162,6 +231,7 @@ setup () { cleanupAndExit 0 fi fi + . $BASEDIR/dev-support/test-patch.properties ### exit if warnings are NOT defined in the properties file if [ -z "$OK_FINDBUGS_WARNINGS" ] || [[ -z "$OK_JAVADOC_WARNINGS" ]] || [[ -z $OK_RELEASEAUDIT_WARNINGS ]]; then echo "Please define the following properties in test-patch.properties file" @@ -179,9 +249,8 @@ setup () { echo "======================================================================" echo "" echo "" -# echo "$ANT_HOME/bin/ant -Djavac.args="-Xlint -Xmaxwarns 1000" $ECLIPSE_PROPERTY -Dforrest.home=${FORREST_HOME} -D${PROJECT_NAME}PatchProcess= clean tar > $PATCH_DIR/trunkJavacWarnings.txt 2>&1" -# $ANT_HOME/bin/ant -Djavac.args="-Xlint -Xmaxwarns 1000" $ECLIPSE_PROPERTY -Dforrest.home=${FORREST_HOME} -D${PROJECT_NAME}PatchProcess= clean tar > $PATCH_DIR/trunkJavacWarnings.txt 2>&1 - $MAVEN_HOME/bin/mvn clean compile -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/trunkJavacWarnings.txt 2>&1 + echo "$MVN clean compile -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/trunkJavacWarnings.txt 2>&1" + $MVN clean compile -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/trunkJavacWarnings.txt 2>&1 if [[ $? != 0 ]] ; then echo "Trunk compilation is broken?" cleanupAndExit 1 @@ -229,7 +298,7 @@ checkTests () { testReferences=`$GREP -c -i '/test' $PATCH_DIR/patch` echo "There appear to be $testReferences test files referenced in the patch." if [[ $testReferences == 0 ]] ; then - if [[ $HUDSON == "true" ]] ; then + if [[ $JENKINS == "true" ]] ; then patchIsDoc=`$GREP -c -i 'title="documentation' $PATCH_DIR/jira` if [[ $patchIsDoc != 0 ]] ; then echo "The patch appears to be a documentation patch that doesn't require tests." @@ -297,12 +366,15 @@ checkJavadocWarnings () { echo "======================================================================" echo "" echo "" - echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= clean javadoc | tee $PATCH_DIR/patchJavadocWarnings.txt" - (cd root; mvn install -DskipTests) - (cd doclet; mvn install -DskipTests) - #$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= clean javadoc | tee $PATCH_DIR/patchJavadocWarnings.txt - $MAVEN_HOME/bin/mvn clean compile javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavadocWarnings.txt 2>&1 - javadocWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/patchJavadocWarnings.txt | awk '/Javadoc Warnings/,EOF' | $GREP -v 'Javadoc Warnings' | awk 'BEGIN {total = 0} {total += 1} END {print total}'` + echo "$MVN clean compile javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavadocWarnings.txt 2>&1" + if [ -d hadoop-project ]; then + (cd hadoop-project; $MVN install) + fi + if [ -d hadoop-annotations ]; then + (cd hadoop-annotations; $MVN install) + fi + $MVN clean compile javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavadocWarnings.txt 2>&1 + javadocWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/patchJavadocWarnings.txt | $AWK '/Javadoc Warnings/,EOF' | $GREP warning | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'` echo "" echo "" echo "There appear to be $javadocWarnings javadoc warnings generated by the patched build." @@ -332,9 +404,8 @@ checkJavacWarnings () { echo "======================================================================" echo "" echo "" - #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -Djavac.args="-Xlint -Xmaxwarns 1000" $ECLIPSE_PROPERTY -Dforrest.home=${FORREST_HOME} -DHadoopPatchProcess= clean tar > $PATCH_DIR/patchJavacWarnings.txt 2>&1" - #$ANT_HOME/bin/ant -Dversion="${VERSION}" -Djavac.args="-Xlint -Xmaxwarns 1000" $ECLIPSE_PROPERTY -Dforrest.home=${FORREST_HOME} -DHadoopPatchProcess= clean tar > $PATCH_DIR/patchJavacWarnings.txt 2>&1 - $MAVEN_HOME/bin/mvn clean compile -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1 + echo "$MVN clean compile -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1" + $MVN clean compile -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1 if [[ $? != 0 ]] ; then JIRA_COMMENT="$JIRA_COMMENT @@ -343,8 +414,8 @@ checkJavacWarnings () { fi ### Compare trunk and patch javac warning numbers if [[ -f $PATCH_DIR/patchJavacWarnings.txt ]] ; then - trunkJavacWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/trunkJavacWarnings.txt | awk 'BEGIN {total = 0} {total += 1} END {print total}'` - patchJavacWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/patchJavacWarnings.txt | awk 'BEGIN {total = 0} {total += 1} END {print total}'` + trunkJavacWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/trunkJavacWarnings.txt | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'` + patchJavacWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/patchJavacWarnings.txt | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'` echo "There appear to be $trunkJavacWarnings javac compiler warnings before the patch and $patchJavacWarnings javac compiler warnings after applying the patch." if [[ $patchJavacWarnings != "" && $trunkJavacWarnings != "" ]] ; then if [[ $patchJavacWarnings -gt $trunkJavacWarnings ]] ; then @@ -373,9 +444,8 @@ checkReleaseAuditWarnings () { echo "======================================================================" echo "" echo "" - #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -Dforrest.home=${FORREST_HOME} -DHadoopPatchProcess= releaseaudit > $PATCH_DIR/patchReleaseAuditWarnings.txt 2>&1" - #$ANT_HOME/bin/ant -Dversion="${VERSION}" -Dforrest.home=${FORREST_HOME} -DHadoopPatchProcess= releaseaudit > $PATCH_DIR/patchReleaseAuditWarnings.txt 2>&1 - $MAVEN_HOME/bin/mvn apache-rat:check -D${PROJECT_NAME}PatchProcess 2>&1 + echo "$MVN apache-rat:check -D${PROJECT_NAME}PatchProcess 2>&1" + $MVN apache-rat:check -D${PROJECT_NAME}PatchProcess 2>&1 find . -name rat.txt | xargs cat > $PATCH_DIR/patchReleaseAuditWarnings.txt ### Compare trunk and patch release audit warning numbers @@ -418,9 +488,8 @@ checkStyle () { echo "THIS IS NOT IMPLEMENTED YET" echo "" echo "" - #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= checkstyle" - #$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= checkstyle - $MAVEN_HOME/bin/mvn compile checkstyle:checkstyle -D${PROJECT_NAME}PatchProcess + echo "$MVN compile checkstyle:checkstyle -D${PROJECT_NAME}PatchProcess" + $MVN compile checkstyle:checkstyle -D${PROJECT_NAME}PatchProcess JIRA_COMMENT_FOOTER="Checkstyle results: $BUILD_URL/artifact/trunk/build/test/checkstyle-errors.html $JIRA_COMMENT_FOOTER" @@ -451,9 +520,8 @@ checkFindbugsWarnings () { echo "======================================================================" echo "" echo "" - #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -Dfindbugs.home=$FINDBUGS_HOME -Dforrest.home=${FORREST_HOME} -DHadoopPatchProcess= findbugs" - #$ANT_HOME/bin/ant -Dversion="${VERSION}" -Dfindbugs.home=${FINDBUGS_HOME} -Dforrest.home=${FORREST_HOME} -DHadoopPatchProcess= findbugs - $MAVEN_HOME/bin/mvn clean compile findbugs:findbugs -D${PROJECT_NAME}PatchProcess -X + echo "$MVN clean compile findbugs:findbugs -D${PROJECT_NAME}PatchProcess" + $MVN clean compile findbugs:findbugs -D${PROJECT_NAME}PatchProcess if [ $? != 0 ] ; then JIRA_COMMENT="$JIRA_COMMENT @@ -461,18 +529,29 @@ checkFindbugsWarnings () { -1 findbugs. The patch appears to cause Findbugs (version ${findbugs_version}) to fail." return 1 fi -JIRA_COMMENT_FOOTER="Findbugs warnings: $BUILD_URL/artifact/trunk/target/newPatchFindbugsWarnings.html + + findbugsWarnings=0 + for file in $(find $BASEDIR -name findbugsXml.xml) + do + relative_file=${file#$BASEDIR/} # strip leading $BASEDIR prefix + if [ ! $relative_file == "target/findbugsXml.xml" ]; then + module_suffix=${relative_file%/target/findbugsXml.xml} # strip trailing path + fi + + cp $file $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml + $FINDBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/2000" \ + $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \ + $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml + newFindbugsWarnings=`$FINDBUGS_HOME/bin/filterBugs -first "01/01/2000" $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \ + $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml | $AWK '{print $1}'` + echo "Found $newFindbugsWarnings Findbugs warnings ($file)" + findbugsWarnings=$((findbugsWarnings+newFindbugsWarnings)) + $FINDBUGS_HOME/bin/convertXmlToText -html \ + $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml \ + $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.html + JIRA_COMMENT_FOOTER="Findbugs warnings: $BUILD_URL/artifact/trunk/target/newPatchFindbugsWarnings${module_suffix}.html $JIRA_COMMENT_FOOTER" - - cp $BASEDIR/hadoop-common/target/findbugsXml.xml $PATCH_DIR/patchFindbugsWarnings.xml - $FINDBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/2000" \ - $PATCH_DIR/patchFindbugsWarnings.xml \ - $PATCH_DIR/patchFindbugsWarnings.xml - findbugsWarnings=`$FINDBUGS_HOME/bin/filterBugs -first "01/01/2000" $PATCH_DIR/patchFindbugsWarnings.xml \ - $PATCH_DIR/newPatchFindbugsWarnings.xml | /usr/bin/awk '{print $1}'` - $FINDBUGS_HOME/bin/convertXmlToText -html \ - $PATCH_DIR/newPatchFindbugsWarnings.xml \ - $PATCH_DIR/newPatchFindbugsWarnings.html + done ### if current warnings greater than OK_FINDBUGS_WARNINGS if [[ $findbugsWarnings > $OK_FINDBUGS_WARNINGS ]] ; then @@ -501,15 +580,14 @@ runCoreTests () { echo "" ### Kill any rogue build processes from the last attempt - $PS auxwww | $GREP HadoopPatchProcess | /usr/bin/nawk '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null + $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null PreTestTarget="" if [[ $defect == MAPREDUCE-* ]] ; then PreTestTarget="create-c++-configure" fi - #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME $PreTestTarget test-core" - #$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME $PreTestTarget test-core - $MAVEN_HOME/bin/mvn clean test -Pnative -DHadoopPatchProcess + echo "$MVN clean test -Pnative -D${PROJECT_NAME}PatchProcess" + $MVN clean test -Pnative -D${PROJECT_NAME}PatchProcess if [[ $? != 0 ]] ; then ### Find and format names of failed tests failed_tests=`grep -l -E " /dev/null + $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" $ECLIPSE_PROPERTY -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no test-contrib" #$ANT_HOME/bin/ant -Dversion="${VERSION}" $ECLIPSE_PROPERTY -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no test-contrib @@ -575,7 +653,7 @@ checkInjectSystemFaults () { echo "" ### Kill any rogue build processes from the last attempt - $PS auxwww | $GREP HadoopPatchProcess | /usr/bin/nawk '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null + $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME inject-system-faults" #$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME inject-system-faults @@ -597,7 +675,7 @@ checkInjectSystemFaults () { submitJiraComment () { local result=$1 ### Do not output the value of JIRA_COMMENT_FOOTER when run by a developer - if [[ $HUDSON == "false" ]] ; then + if [[ $JENKINS == "false" ]] ; then JIRA_COMMENT_FOOTER="" fi if [[ $result == 0 ]] ; then @@ -616,7 +694,7 @@ $JIRA_COMMENT_FOOTER" $comment" - if [[ $HUDSON == "true" ]] ; then + if [[ $JENKINS == "true" ]] ; then echo "" echo "" echo "======================================================================" @@ -637,7 +715,7 @@ $comment" ### Cleanup files cleanupAndExit () { local result=$1 - if [[ $HUDSON == "true" ]] ; then + if [[ $JENKINS == "true" ]] ; then if [ -e "$PATCH_DIR" ] ; then mv $PATCH_DIR $BASEDIR fi @@ -669,7 +747,7 @@ cd $BASEDIR checkout RESULT=$? -if [[ $HUDSON == "true" ]] ; then +if [[ $JENKINS == "true" ]] ; then if [[ $RESULT != 0 ]] ; then exit 100 fi @@ -678,7 +756,7 @@ setup checkAuthor RESULT=$? -if [[ $HUDSON == "true" ]] ; then +if [[ $JENKINS == "true" ]] ; then cleanUpXml fi checkTests @@ -700,7 +778,7 @@ checkFindbugsWarnings checkReleaseAuditWarnings (( RESULT = RESULT + $? )) ### Do not call these when run by a developer -if [[ $HUDSON == "true" ]] ; then +if [[ $JENKINS == "true" ]] ; then runCoreTests (( RESULT = RESULT + $? )) runContribTests diff --git a/hadoop-assemblies/pom.xml b/hadoop-assemblies/pom.xml index 59062a4b78c..f6feb3a1432 100644 --- a/hadoop-assemblies/pom.xml +++ b/hadoop-assemblies/pom.xml @@ -67,9 +67,6 @@ 1.6 - - unix - diff --git a/hadoop-common/CHANGES.txt b/hadoop-common/CHANGES.txt index c499c750f09..f8b24db420f 100644 --- a/hadoop-common/CHANGES.txt +++ b/hadoop-common/CHANGES.txt @@ -309,11 +309,29 @@ Trunk (unreleased changes) HADOOP-7501. Publish Hadoop Common artifacts (post HADOOP-6671) to Apache SNAPSHOTs repo. (Alejandro Abdelnur via tomwhite) + HADOOP-7525. Make arguments to test-patch optional. (tomwhite) + + HADOOP-7472. RPC client should deal with IP address change. + (Kihwal Lee via suresh) + + HADOOP-7499. Add method for doing a sanity check on hostnames in NetUtils. + (Jeffrey Naisbit via mahadev) + + HADOOP-6158. Move CyclicIteration to HDFS. (eli) + + HADOOP-7526. Add TestPath tests for URI conversion and reserved + characters. (eli) + + HADOOP-7531. Add servlet util methods for handling paths in requests. (eli) + OPTIMIZATIONS HADOOP-7333. Performance improvement in PureJavaCrc32. (Eric Caspole via todd) + HADOOP-7445. Implement bulk checksum verification using efficient native + code. (todd) + BUG FIXES HADOOP-7327. FileSystem.listStatus() throws NullPointerException instead of @@ -451,6 +469,34 @@ Trunk (unreleased changes) HADOOP-7508. Compiled nativelib is in wrong directory and it is not picked up by surefire setup. (Alejandro Abdelnur via tomwhite) + + HADOOP-7520. Fix to add distribution management info to hadoop-main + (Alejandro Abdelnur via gkesavan) + + HADOOP-7515. test-patch reports the wrong number of javadoc warnings. + (tomwhite) + + HADOOP-7523. Test org.apache.hadoop.fs.TestFilterFileSystem fails due to + java.lang.NoSuchMethodException. (John Lee via tomwhite) + + HADOOP-7528. Maven build fails in Windows. (Alejandro Abdelnur via + tomwhite) + + HADOOP-7533. Allow test-patch to be run from any subproject directory. + (tomwhite) + + HADOOP-7512. Fix example mistake in WritableComparable javadocs. + (Harsh J via eli) + + HADOOP-7357. hadoop.io.compress.TestCodec#main() should exit with + non-zero exit code if test failed. (Philip Zeyliger via eli) + + HADOOP-6622. Token should not print the password in toString. (eli) + + HADOOP-7529. Fix lock cycles in metrics system. (llu) + + HADOOP-7545. Common -tests JAR should not include properties and configs. + (todd) Release 0.22.0 - Unreleased diff --git a/hadoop-common/LICENSE.txt b/hadoop-common/LICENSE.txt index 59bcdbc9783..111653695f0 100644 --- a/hadoop-common/LICENSE.txt +++ b/hadoop-common/LICENSE.txt @@ -242,3 +242,12 @@ For the org.apache.hadoop.util.bloom.* classes: * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ + +For portions of the native implementation of slicing-by-8 CRC calculation +in src/main/native/src/org/apache/hadoop/util: + +/** + * Copyright 2008,2009,2010 Massachusetts Institute of Technology. + * All rights reserved. Use of this source code is governed by a + * BSD-style license that can be found in the LICENSE file. + */ diff --git a/hadoop-common/dev-support/test-patch.properties b/hadoop-common/dev-support/test-patch.properties new file mode 100644 index 00000000000..15b54bfcf0d --- /dev/null +++ b/hadoop-common/dev-support/test-patch.properties @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The number of acceptable warning for this module +# Please update the root test-patch.properties if you update this file. + +OK_RELEASEAUDIT_WARNINGS=0 +OK_FINDBUGS_WARNINGS=0 +OK_JAVADOC_WARNINGS=6 diff --git a/hadoop-common/pom.xml b/hadoop-common/pom.xml index d5635b0c8e0..eb10c0d50c2 100644 --- a/hadoop-common/pom.xml +++ b/hadoop-common/pom.xml @@ -279,11 +279,23 @@ maven-jar-plugin + prepare-jar prepare-package jar + + + + prepare-test-jar + prepare-package + test-jar + + + **/*.class + + @@ -545,6 +557,7 @@ org.apache.hadoop.security.JniBasedUnixGroupsNetgroupMapping org.apache.hadoop.io.compress.snappy.SnappyCompressor org.apache.hadoop.io.compress.snappy.SnappyDecompressor + org.apache.hadoop.util.NativeCrc32 ${project.build.directory}/native/javah @@ -852,11 +865,17 @@ + which cygpath 2> /dev/null + if [ $? = 1 ]; then + BUILD_DIR="${project.build.directory}" + else + BUILD_DIR=`cygpath --unix '${project.build.directory}'` + fi TAR='tar cf -' UNTAR='tar xfBp -' - LIB_DIR="${project.build.directory}/native/target/usr/local/lib" + LIB_DIR="${BUILD_DIR}/native/target/usr/local/lib" if [ -d $${LIB_DIR} ] ; then - TARGET_DIR="${project.build.directory}/${project.artifactId}-${project.version}/lib/native/${build.platform}" + TARGET_DIR="${BUILD_DIR}/${project.artifactId}-${project.version}/lib/native/${build.platform}" mkdir -p $${TARGET_DIR} cd $${LIB_DIR} $$TAR *hadoop* | (cd $${TARGET_DIR}/; $$UNTAR) @@ -880,11 +899,20 @@ - - - - - + + + + which cygpath 2> /dev/null + if [ $? = 1 ]; then + BUILD_DIR="${project.build.directory}" + else + BUILD_DIR=`cygpath --unix '${project.build.directory}'` + fi + cd ${BUILD_DIR} + tar czf ${project.artifactId}-${project.version}.tar.gz ${project.artifactId}-${project.version} + + + @@ -945,11 +973,17 @@ + which cygpath 2> /dev/null + if [ $? = 1 ]; then + BUILD_DIR="${project.build.directory}" + else + BUILD_DIR=`cygpath --unix '${project.build.directory}'` + fi TAR='tar cf -' UNTAR='tar xfBp -' - LIB_DIR="${project.build.directory}/native/target/usr/local/lib" + LIB_DIR="${BUILD_DIR}/native/target/usr/local/lib" if [ -d $${LIB_DIR} ] ; then - TARGET_DIR="${project.build.directory}/${project.artifactId}-${project.version}-bin/lib" + TARGET_DIR="${BUILD_DIR}/${project.artifactId}-${project.version}-bin/lib" mkdir -p $${TARGET_DIR} cd $${LIB_DIR} $$TAR *hadoop* | (cd $${TARGET_DIR}/; $$UNTAR) @@ -973,11 +1007,20 @@ - - - - - + + + + which cygpath 2> /dev/null + if [ $? = 1 ]; then + BUILD_DIR="${project.build.directory}" + else + BUILD_DIR=`cygpath --unix '${project.build.directory}'` + fi + cd ${BUILD_DIR} + tar czf ${project.artifactId}-${project.version}-bin.tar.gz ${project.artifactId}-${project.version}-bin + + + diff --git a/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparable.java b/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparable.java index 021b2b5c00a..7fd83e31905 100644 --- a/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparable.java +++ b/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparable.java @@ -52,7 +52,7 @@ import org.apache.hadoop.classification.InterfaceStability; * timestamp = in.readLong(); * } * - * public int compareTo(MyWritableComparable w) { + * public int compareTo(MyWritableComparable o) { * int thisValue = this.value; * int thatValue = ((IntWritable)o).value; * return (thisValue < thatValue ? -1 : (thisValue==thatValue ? 0 : 1)); diff --git a/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java index fe719de0817..9c35e8e9c48 100644 --- a/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java +++ b/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java @@ -405,6 +405,27 @@ public class Client { saslRpcClient = new SaslRpcClient(authMethod, token, serverPrincipal); return saslRpcClient.saslConnect(in2, out2); } + + /** + * Update the server address if the address corresponding to the host + * name has changed. + * + * @return true if an addr change was detected. + * @throws IOException when the hostname cannot be resolved. + */ + private synchronized boolean updateAddress() throws IOException { + // Do a fresh lookup with the old host name. + InetSocketAddress currentAddr = new InetSocketAddress( + server.getHostName(), server.getPort()); + + if (!server.equals(currentAddr)) { + LOG.warn("Address change detected. Old: " + server.toString() + + " New: " + currentAddr.toString()); + server = currentAddr; + return true; + } + return false; + } private synchronized void setupConnection() throws IOException { short ioFailures = 0; @@ -435,19 +456,28 @@ public class Client { } // connection time out is 20s - NetUtils.connect(this.socket, remoteId.getAddress(), 20000); + NetUtils.connect(this.socket, server, 20000); if (rpcTimeout > 0) { pingInterval = rpcTimeout; // rpcTimeout overwrites pingInterval } this.socket.setSoTimeout(pingInterval); return; } catch (SocketTimeoutException toe) { + /* Check for an address change and update the local reference. + * Reset the failure counter if the address was changed + */ + if (updateAddress()) { + timeoutFailures = ioFailures = 0; + } /* * The max number of retries is 45, which amounts to 20s*45 = 15 * minutes retries. */ handleConnectionFailure(timeoutFailures++, 45, toe); } catch (IOException ie) { + if (updateAddress()) { + timeoutFailures = ioFailures = 0; + } handleConnectionFailure(ioFailures++, maxRetries, ie); } } diff --git a/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystem.java b/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystem.java index 12ed2e28878..c8b33bc67f8 100644 --- a/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystem.java +++ b/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystem.java @@ -18,6 +18,7 @@ package org.apache.hadoop.metrics2.lib; +import java.util.concurrent.atomic.AtomicReference; import javax.management.ObjectName; import org.apache.hadoop.classification.InterfaceAudience; @@ -34,7 +35,8 @@ import org.apache.hadoop.metrics2.impl.MetricsSystemImpl; public enum DefaultMetricsSystem { INSTANCE; // the singleton - private MetricsSystem impl = new MetricsSystemImpl(); + private AtomicReference impl = + new AtomicReference(new MetricsSystemImpl()); volatile boolean miniClusterMode = false; final UniqueNames mBeanNames = new UniqueNames(); final UniqueNames sourceNames = new UniqueNames(); @@ -48,8 +50,8 @@ public enum DefaultMetricsSystem { return INSTANCE.init(prefix); } - synchronized MetricsSystem init(String prefix) { - return impl.init(prefix); + MetricsSystem init(String prefix) { + return impl.get().init(prefix); } /** @@ -66,8 +68,9 @@ public enum DefaultMetricsSystem { INSTANCE.shutdownInstance(); } - synchronized void shutdownInstance() { - if (impl.shutdown()) { + void shutdownInstance() { + boolean last = impl.get().shutdown(); + if (last) synchronized(this) { mBeanNames.map.clear(); sourceNames.map.clear(); } @@ -78,13 +81,11 @@ public enum DefaultMetricsSystem { return INSTANCE.setImpl(ms); } - synchronized MetricsSystem setImpl(MetricsSystem ms) { - MetricsSystem old = impl; - impl = ms; - return old; + MetricsSystem setImpl(MetricsSystem ms) { + return impl.getAndSet(ms); } - synchronized MetricsSystem getImpl() { return impl; } + MetricsSystem getImpl() { return impl.get(); } @InterfaceAudience.Private public static void setMiniClusterMode(boolean choice) { diff --git a/hadoop-common/src/main/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java b/hadoop-common/src/main/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java index a9d6b7f2dff..5bd3ca35de8 100644 --- a/hadoop-common/src/main/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java +++ b/hadoop-common/src/main/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java @@ -17,9 +17,6 @@ */ package org.apache.hadoop.net; -import java.net.InetAddress; -import java.net.SocketException; -import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -91,32 +88,6 @@ public class CachedDNSToSwitchMapping implements DNSToSwitchMapping { return result; } - /** - * Resolves host names and adds them to the cache. - * Unlike the 'resolve" method, this won't hide UnknownHostExceptions - * - * @param names to resolve - * @return List of resolved names - * @throws UnknownHostException if any hosts cannot be resolved - */ - public List resolveValidHosts(List names) - throws UnknownHostException { - if (names.isEmpty()) { - return new ArrayList(); - } - List addresses = new ArrayList(names.size()); - for (String name : names) { - addresses.add(InetAddress.getByName(name).getHostAddress()); - } - - List uncachedHosts = this.getUncachedHosts(names); - - // Resolve the uncached hosts - List resolvedHosts = rawMapping.resolveValidHosts(uncachedHosts); - this.cacheResolvedHosts(uncachedHosts, resolvedHosts); - return this.getCachedHosts(addresses); - } - public List resolve(List names) { // normalize all input names to be in the form of IP addresses names = NetUtils.normalizeHostNames(names); diff --git a/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java b/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java index 3f0f7b3834a..f9816becb6c 100644 --- a/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java +++ b/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java @@ -18,7 +18,6 @@ package org.apache.hadoop.net; import java.util.List; -import java.net.UnknownHostException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -45,23 +44,4 @@ public interface DNSToSwitchMapping { * @return list of resolved network paths */ public List resolve(List names); - - /** - * Resolves a list of DNS-names/IP-addresses and returns back a list of - * switch information (network paths). One-to-one correspondence must be - * maintained between the elements in the lists. - * Consider an element in the argument list - x.y.com. The switch information - * that is returned must be a network path of the form /foo/rack, - * where / is the root, and 'foo' is the switch where 'rack' is connected. - * Note the hostname/ip-address is not part of the returned path. - * The network topology of the cluster would determine the number of - * components in the network path. Unlike 'resolve', names must be - * resolvable - * @param names - * @return list of resolved network paths - * @throws UnknownHostException if any hosts are not resolvable - */ - public List resolveValidHosts(List names) - throws UnknownHostException; - } diff --git a/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java b/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java index 4a9a7fe1a1b..b22aaa009c1 100644 --- a/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java +++ b/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java @@ -27,6 +27,7 @@ import java.net.Socket; import java.net.SocketAddress; import java.net.SocketException; import java.net.URI; +import java.net.URISyntaxException; import java.net.UnknownHostException; import java.net.ConnectException; import java.nio.channels.SocketChannel; @@ -428,6 +429,35 @@ public class NetUtils { return hostNames; } + /** + * Performs a sanity check on the list of hostnames/IPs to verify they at least + * appear to be valid. + * @param names - List of hostnames/IPs + * @throws UnknownHostException + */ + public static void verifyHostnames(String[] names) throws UnknownHostException { + for (String name: names) { + if (name == null) { + throw new UnknownHostException("null hostname found"); + } + // The first check supports URL formats (e.g. hdfs://, etc.). + // java.net.URI requires a schema, so we add a dummy one if it doesn't + // have one already. + URI uri = null; + try { + uri = new URI(name); + if (uri.getHost() == null) { + uri = new URI("http://" + name); + } + } catch (URISyntaxException e) { + uri = null; + } + if (uri == null || uri.getHost() == null) { + throw new UnknownHostException(name + " is not a valid Inet address"); + } + } + } + private static final Pattern ipPattern = // Pattern for matching hostname to ip:port Pattern.compile("\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}:?\\d*"); diff --git a/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java b/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java index 85dbe201aa0..e3fe4581bb1 100644 --- a/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java +++ b/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java @@ -20,7 +20,6 @@ package org.apache.hadoop.net; import java.util.*; import java.io.*; -import java.net.UnknownHostException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -123,17 +122,6 @@ implements Configurable return m; } - - public List resolveValidHosts(List names) - throws UnknownHostException { - List result = this.resolve(names); - if (result != null) { - return result; - } else { - throw new UnknownHostException( - "Unknown host(s) returned from ScriptBasedMapping"); - } - } private String runResolveCommand(List args) { int loopCount = 0; diff --git a/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java b/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java index eac07c5f75f..d47f99429f4 100644 --- a/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java +++ b/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java @@ -238,8 +238,6 @@ public class Token implements Writable { StringBuilder buffer = new StringBuilder(); buffer.append("Ident: "); addBinaryBuffer(buffer, identifier); - buffer.append(", Pass: "); - addBinaryBuffer(buffer, password); buffer.append(", Kind: "); buffer.append(kind.toString()); buffer.append(", Service: "); diff --git a/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java b/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java index 5ef3062d302..b227a65fbe4 100644 --- a/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java +++ b/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java @@ -265,6 +265,11 @@ public class DataChecksum implements Checksum { fileName, basePos); return; } + if (NativeCrc32.isAvailable()) { + NativeCrc32.verifyChunkedSums(bytesPerChecksum, type, checksums, data, + fileName, basePos); + return; + } int startDataPos = data.position(); data.mark(); diff --git a/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCrc32.java b/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCrc32.java new file mode 100644 index 00000000000..97919ad92a2 --- /dev/null +++ b/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCrc32.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.util; + +import java.nio.ByteBuffer; + +import org.apache.hadoop.fs.ChecksumException; + +/** + * Wrapper around JNI support code to do checksum computation + * natively. + */ +class NativeCrc32 { + + /** + * Return true if the JNI-based native CRC extensions are available. + */ + public static boolean isAvailable() { + return NativeCodeLoader.isNativeCodeLoaded(); + } + + /** + * Verify the given buffers of data and checksums, and throw an exception + * if any checksum is invalid. The buffers given to this function should + * have their position initially at the start of the data, and their limit + * set at the end of the data. The position, limit, and mark are not + * modified. + * + * @param bytesPerSum the chunk size (eg 512 bytes) + * @param checksumType the DataChecksum type constant + * @param sums the DirectByteBuffer pointing at the beginning of the + * stored checksums + * @param data the DirectByteBuffer pointing at the beginning of the + * data to check + * @param basePos the position in the file where the data buffer starts + * @param fileName the name of the file being verified + * @throws ChecksumException if there is an invalid checksum + */ + public static void verifyChunkedSums(int bytesPerSum, int checksumType, + ByteBuffer sums, ByteBuffer data, String fileName, long basePos) + throws ChecksumException { + nativeVerifyChunkedSums(bytesPerSum, checksumType, + sums, sums.position(), + data, data.position(), data.remaining(), + fileName, basePos); + } + + private static native void nativeVerifyChunkedSums( + int bytesPerSum, int checksumType, + ByteBuffer sums, int sumsOffset, + ByteBuffer data, int dataOffset, int dataLength, + String fileName, long basePos); + + // Copy the constants over from DataChecksum so that javah will pick them up + // and make them available in the native code header. + public static final int CHECKSUM_CRC32 = DataChecksum.CHECKSUM_CRC32; + public static final int CHECKSUM_CRC32C = DataChecksum.CHECKSUM_CRC32C; +} diff --git a/hadoop-common/src/main/java/org/apache/hadoop/util/ServletUtil.java b/hadoop-common/src/main/java/org/apache/hadoop/util/ServletUtil.java index 9d801cd243b..993394d59bf 100644 --- a/hadoop-common/src/main/java/org/apache/hadoop/util/ServletUtil.java +++ b/hadoop-common/src/main/java/org/apache/hadoop/util/ServletUtil.java @@ -21,10 +21,15 @@ import java.io.*; import java.util.Calendar; import javax.servlet.*; +import javax.servlet.http.HttpServletRequest; +import org.apache.commons.httpclient.URIException; +import org.apache.commons.httpclient.util.URIUtil; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import com.google.common.base.Preconditions; + @InterfaceAudience.Private @InterfaceStability.Unstable public class ServletUtil { @@ -107,4 +112,55 @@ public class ServletUtil { public static String percentageGraph(float perc, int width) throws IOException { return percentageGraph((int)perc, width); } -} + + /** + * Escape and encode a string regarded as within the query component of an URI. + * @param value the value to encode + * @return encoded query, null if the default charset is not supported + */ + public static String encodeQueryValue(final String value) { + try { + return URIUtil.encodeWithinQuery(value, "UTF-8"); + } catch (URIException e) { + throw new AssertionError("JVM does not support UTF-8"); // should never happen! + } + } + + /** + * Escape and encode a string regarded as the path component of an URI. + * @param path the path component to encode + * @return encoded path, null if UTF-8 is not supported + */ + public static String encodePath(final String path) { + try { + return URIUtil.encodePath(path, "UTF-8"); + } catch (URIException e) { + throw new AssertionError("JVM does not support UTF-8"); // should never happen! + } + } + + /** + * Parse and decode the path component from the given request. + * @param request Http request to parse + * @param servletName the name of servlet that precedes the path + * @return decoded path component, null if UTF-8 is not supported + */ + public static String getDecodedPath(final HttpServletRequest request, String servletName) { + try { + return URIUtil.decode(getRawPath(request, servletName), "UTF-8"); + } catch (URIException e) { + throw new AssertionError("JVM does not support UTF-8"); // should never happen! + } + } + + /** + * Parse the path component from the given request and return w/o decoding. + * @param request Http request to parse + * @param servletName the name of servlet that precedes the path + * @return path component, null if the default charset is not supported + */ + public static String getRawPath(final HttpServletRequest request, String servletName) { + Preconditions.checkArgument(request.getRequestURI().startsWith(servletName+"/")); + return request.getRequestURI().substring(servletName.length()); + } +} \ No newline at end of file diff --git a/hadoop-common/src/main/native/Makefile.am b/hadoop-common/src/main/native/Makefile.am index d981641730d..3afc0b88a02 100644 --- a/hadoop-common/src/main/native/Makefile.am +++ b/hadoop-common/src/main/native/Makefile.am @@ -51,7 +51,9 @@ libhadoop_la_SOURCES = src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c \ src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c \ src/org/apache/hadoop/io/nativeio/file_descriptor.c \ src/org/apache/hadoop/io/nativeio/errno_enum.c \ - src/org/apache/hadoop/io/nativeio/NativeIO.c + src/org/apache/hadoop/io/nativeio/NativeIO.c \ + src/org/apache/hadoop/util/NativeCrc32.c \ + src/org/apache/hadoop/util/bulk_crc32.c libhadoop_la_LDFLAGS = -version-info 1:0:0 $(AM_LDFLAGS) libhadoop_la_LIBADD = -ldl -ljvm diff --git a/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c b/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c new file mode 100644 index 00000000000..cba6c7c53ef --- /dev/null +++ b/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// get the autoconf settings +#include "config.h" + +#include +#include +#include +#include +#include +#include + +#include "org_apache_hadoop.h" +#include "org_apache_hadoop_util_NativeCrc32.h" +#include "gcc_optimizations.h" +#include "bulk_crc32.h" + +static void throw_checksum_exception(JNIEnv *env, + uint32_t got_crc, uint32_t expected_crc, + jstring j_filename, jlong pos) { + char message[1024]; + jstring jstr_message; + char *filename; + + // Get filename as C string, or "null" if not provided + if (j_filename == NULL) { + filename = strdup("null"); + } else { + const char *c_filename = (*env)->GetStringUTFChars(env, j_filename, NULL); + if (c_filename == NULL) { + return; // OOME already thrown + } + filename = strdup(c_filename); + (*env)->ReleaseStringUTFChars(env, j_filename, c_filename); + } + + // Format error message + snprintf(message, sizeof(message), + "Checksum error: %s at %ld exp: %d got: %d", + filename, pos, expected_crc, got_crc); + if ((jstr_message = (*env)->NewStringUTF(env, message)) == NULL) { + goto cleanup; + } + + // Throw exception + jclass checksum_exception_clazz = (*env)->FindClass( + env, "org/apache/hadoop/fs/ChecksumException"); + if (checksum_exception_clazz == NULL) { + goto cleanup; + } + + jmethodID checksum_exception_ctor = (*env)->GetMethodID(env, + checksum_exception_clazz, "", + "(Ljava/lang/String;J)V"); + if (checksum_exception_ctor == NULL) { + goto cleanup; + } + + jthrowable obj = (jthrowable)(*env)->NewObject(env, checksum_exception_clazz, + checksum_exception_ctor, jstr_message, pos); + if (obj == NULL) goto cleanup; + + (*env)->Throw(env, obj); + +cleanup: + if (filename != NULL) { + free(filename); + } +} + +static int convert_java_crc_type(JNIEnv *env, jint crc_type) { + switch (crc_type) { + case org_apache_hadoop_util_NativeCrc32_CHECKSUM_CRC32: + return CRC32_ZLIB_POLYNOMIAL; + case org_apache_hadoop_util_NativeCrc32_CHECKSUM_CRC32C: + return CRC32C_POLYNOMIAL; + default: + THROW(env, "java/lang/IllegalArgumentException", + "Invalid checksum type"); + return -1; + } +} + +JNIEXPORT void JNICALL Java_org_apache_hadoop_util_NativeCrc32_nativeVerifyChunkedSums + (JNIEnv *env, jclass clazz, + jint bytes_per_checksum, jint j_crc_type, + jobject j_sums, jint sums_offset, + jobject j_data, jint data_offset, jint data_len, + jstring j_filename, jlong base_pos) +{ + if (unlikely(!j_sums || !j_data)) { + THROW(env, "java/lang/NullPointerException", + "input ByteBuffers must not be null"); + return; + } + + // Convert direct byte buffers to C pointers + uint8_t *sums_addr = (*env)->GetDirectBufferAddress(env, j_sums); + uint8_t *data_addr = (*env)->GetDirectBufferAddress(env, j_data); + + if (unlikely(!sums_addr || !data_addr)) { + THROW(env, "java/lang/IllegalArgumentException", + "input ByteBuffers must be direct buffers"); + return; + } + if (unlikely(sums_offset < 0 || data_offset < 0 || data_len < 0)) { + THROW(env, "java/lang/IllegalArgumentException", + "bad offsets or lengths"); + return; + } + + uint32_t *sums = (uint32_t *)(sums_addr + sums_offset); + uint8_t *data = data_addr + data_offset; + + // Convert to correct internal C constant for CRC type + int crc_type = convert_java_crc_type(env, j_crc_type); + if (crc_type == -1) return; // exception already thrown + + // Setup complete. Actually verify checksums. + crc32_error_t error_data; + int ret = bulk_verify_crc(data, data_len, sums, crc_type, + bytes_per_checksum, &error_data); + if (likely(ret == CHECKSUMS_VALID)) { + return; + } else if (unlikely(ret == INVALID_CHECKSUM_DETECTED)) { + long pos = base_pos + (error_data.bad_data - data); + throw_checksum_exception( + env, error_data.got_crc, error_data.expected_crc, + j_filename, pos); + } else { + THROW(env, "java/lang/AssertionError", + "Bad response code from native bulk_verify_crc"); + } +} + +/** + * vim: sw=2: ts=2: et: + */ diff --git a/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.c b/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.c new file mode 100644 index 00000000000..4a769b41edb --- /dev/null +++ b/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.c @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Portions of this file are from http://www.evanjones.ca/crc32c.html under + * the BSD license: + * Copyright 2008,2009,2010 Massachusetts Institute of Technology. + * All rights reserved. Use of this source code is governed by a + * BSD-style license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "crc32_zlib_polynomial_tables.h" +#include "crc32c_tables.h" +#include "bulk_crc32.h" +#include "gcc_optimizations.h" + +typedef uint32_t (*crc_update_func_t)(uint32_t, const uint8_t *, size_t); +static uint32_t crc_init(); +static uint32_t crc_val(uint32_t crc); +static uint32_t crc32_zlib_sb8(uint32_t crc, const uint8_t *buf, size_t length); +static uint32_t crc32c_sb8(uint32_t crc, const uint8_t *buf, size_t length); + +int bulk_verify_crc(const uint8_t *data, size_t data_len, + const uint32_t *sums, int checksum_type, + int bytes_per_checksum, + crc32_error_t *error_info) { + + crc_update_func_t crc_update_func; + switch (checksum_type) { + case CRC32_ZLIB_POLYNOMIAL: + crc_update_func = crc32_zlib_sb8; + break; + case CRC32C_POLYNOMIAL: + crc_update_func = crc32c_sb8; + break; + default: + return INVALID_CHECKSUM_TYPE; + } + + while (likely(data_len > 0)) { + int len = likely(data_len >= bytes_per_checksum) ? bytes_per_checksum : data_len; + uint32_t crc = crc_init(); + crc = crc_update_func(crc, data, len); + crc = ntohl(crc_val(crc)); + if (unlikely(crc != *sums)) { + if (error_info != NULL) { + error_info->got_crc = crc; + error_info->expected_crc = *sums; + error_info->bad_data = data; + } + return INVALID_CHECKSUM_DETECTED; + } + data += len; + data_len -= len; + sums++; + } + return CHECKSUMS_VALID; +} + + +/** + * Initialize a CRC + */ +static uint32_t crc_init() { + return 0xffffffff; +} + +/** + * Extract the final result of a CRC + */ +static uint32_t crc_val(uint32_t crc) { + return ~crc; +} + +/** + * Computes the CRC32c checksum for the specified buffer using the slicing by 8 + * algorithm over 64 bit quantities. + */ +static uint32_t crc32c_sb8(uint32_t crc, const uint8_t *buf, size_t length) { + uint32_t running_length = ((length)/8)*8; + uint32_t end_bytes = length - running_length; + int li; + for (li=0; li < running_length/8; li++) { + crc ^= *(uint32_t *)buf; + buf += 4; + uint32_t term1 = CRC32C_T8_7[crc & 0x000000FF] ^ + CRC32C_T8_6[(crc >> 8) & 0x000000FF]; + uint32_t term2 = crc >> 16; + crc = term1 ^ + CRC32C_T8_5[term2 & 0x000000FF] ^ + CRC32C_T8_4[(term2 >> 8) & 0x000000FF]; + term1 = CRC32C_T8_3[(*(uint32_t *)buf) & 0x000000FF] ^ + CRC32C_T8_2[((*(uint32_t *)buf) >> 8) & 0x000000FF]; + + term2 = (*(uint32_t *)buf) >> 16; + crc = crc ^ + term1 ^ + CRC32C_T8_1[term2 & 0x000000FF] ^ + CRC32C_T8_0[(term2 >> 8) & 0x000000FF]; + buf += 4; + } + for (li=0; li < end_bytes; li++) { + crc = CRC32C_T8_0[(crc ^ *buf++) & 0x000000FF] ^ (crc >> 8); + } + return crc; +} + +/** + * Update a CRC using the "zlib" polynomial -- what Hadoop calls CHECKSUM_CRC32 + * using slicing-by-8 + */ +static uint32_t crc32_zlib_sb8( + uint32_t crc, const uint8_t *buf, size_t length) { + uint32_t running_length = ((length)/8)*8; + uint32_t end_bytes = length - running_length; + int li; + for (li=0; li < running_length/8; li++) { + crc ^= *(uint32_t *)buf; + buf += 4; + uint32_t term1 = CRC32_T8_7[crc & 0x000000FF] ^ + CRC32_T8_6[(crc >> 8) & 0x000000FF]; + uint32_t term2 = crc >> 16; + crc = term1 ^ + CRC32_T8_5[term2 & 0x000000FF] ^ + CRC32_T8_4[(term2 >> 8) & 0x000000FF]; + term1 = CRC32_T8_3[(*(uint32_t *)buf) & 0x000000FF] ^ + CRC32_T8_2[((*(uint32_t *)buf) >> 8) & 0x000000FF]; + + term2 = (*(uint32_t *)buf) >> 16; + crc = crc ^ + term1 ^ + CRC32_T8_1[term2 & 0x000000FF] ^ + CRC32_T8_0[(term2 >> 8) & 0x000000FF]; + buf += 4; + } + for (li=0; li < end_bytes; li++) { + crc = CRC32_T8_0[(crc ^ *buf++) & 0x000000FF] ^ (crc >> 8); + } + return crc; +} diff --git a/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.h b/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.h new file mode 100644 index 00000000000..2ab1bd3c402 --- /dev/null +++ b/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.h @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef BULK_CRC32_H_INCLUDED +#define BULK_CRC32_H_INCLUDED + +#include + +// Constants for different CRC algorithms +#define CRC32C_POLYNOMIAL 1 +#define CRC32_ZLIB_POLYNOMIAL 2 + +// Return codes for bulk_verify_crc +#define CHECKSUMS_VALID 0 +#define INVALID_CHECKSUM_DETECTED -1 +#define INVALID_CHECKSUM_TYPE -2 + +// Return type for bulk verification when verification fails +typedef struct crc32_error { + uint32_t got_crc; + uint32_t expected_crc; + const uint8_t *bad_data; // pointer to start of data chunk with error +} crc32_error_t; + + +/** + * Verify a buffer of data which is checksummed in chunks + * of bytes_per_checksum bytes. The checksums are each 32 bits + * and are stored in sequential indexes of the 'sums' array. + * + * checksum_type - one of the CRC32 constants defined above + * error_info - if non-NULL, will be filled in if an error + * is detected + * + * Returns: 0 for success, non-zero for an error, result codes + * for which are defined above + */ +extern int bulk_verify_crc(const uint8_t *data, size_t data_len, + const uint32_t *sums, int checksum_type, + int bytes_per_checksum, + crc32_error_t *error_info); + +#endif diff --git a/hadoop-common/src/main/native/src/org/apache/hadoop/util/crc32_zlib_polynomial_tables.h b/hadoop-common/src/main/native/src/org/apache/hadoop/util/crc32_zlib_polynomial_tables.h new file mode 100644 index 00000000000..59d8f4dfc66 --- /dev/null +++ b/hadoop-common/src/main/native/src/org/apache/hadoop/util/crc32_zlib_polynomial_tables.h @@ -0,0 +1,552 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * CRC-32 lookup tables generated by the polynomial 0xEDB88320. + * See also TestPureJavaCrc32.Table. + */ +const uint32_t CRC32_T8_0[] = { + 0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, + 0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3, + 0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, + 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91, + 0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE, + 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7, + 0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC, + 0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5, + 0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172, + 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B, + 0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940, + 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59, + 0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116, + 0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F, + 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924, + 0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D, + 0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A, + 0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433, + 0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818, + 0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01, + 0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E, + 0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457, + 0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA, 0xFCB9887C, + 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65, + 0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2, + 0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB, + 0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0, + 0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9, + 0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086, + 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F, + 0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, + 0x59B33D17, 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD, + 0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A, + 0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683, + 0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8, + 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1, + 0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE, + 0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7, + 0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC, + 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5, + 0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252, + 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B, + 0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60, + 0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79, + 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236, + 0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F, + 0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04, + 0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D, + 0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A, + 0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713, + 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38, + 0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21, + 0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E, + 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777, + 0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C, + 0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45, + 0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2, + 0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB, + 0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0, + 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9, + 0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, + 0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF, + 0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94, + 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D +}; +const uint32_t CRC32_T8_1[] = { + 0x00000000, 0x191B3141, 0x32366282, 0x2B2D53C3, + 0x646CC504, 0x7D77F445, 0x565AA786, 0x4F4196C7, + 0xC8D98A08, 0xD1C2BB49, 0xFAEFE88A, 0xE3F4D9CB, + 0xACB54F0C, 0xB5AE7E4D, 0x9E832D8E, 0x87981CCF, + 0x4AC21251, 0x53D92310, 0x78F470D3, 0x61EF4192, + 0x2EAED755, 0x37B5E614, 0x1C98B5D7, 0x05838496, + 0x821B9859, 0x9B00A918, 0xB02DFADB, 0xA936CB9A, + 0xE6775D5D, 0xFF6C6C1C, 0xD4413FDF, 0xCD5A0E9E, + 0x958424A2, 0x8C9F15E3, 0xA7B24620, 0xBEA97761, + 0xF1E8E1A6, 0xE8F3D0E7, 0xC3DE8324, 0xDAC5B265, + 0x5D5DAEAA, 0x44469FEB, 0x6F6BCC28, 0x7670FD69, + 0x39316BAE, 0x202A5AEF, 0x0B07092C, 0x121C386D, + 0xDF4636F3, 0xC65D07B2, 0xED705471, 0xF46B6530, + 0xBB2AF3F7, 0xA231C2B6, 0x891C9175, 0x9007A034, + 0x179FBCFB, 0x0E848DBA, 0x25A9DE79, 0x3CB2EF38, + 0x73F379FF, 0x6AE848BE, 0x41C51B7D, 0x58DE2A3C, + 0xF0794F05, 0xE9627E44, 0xC24F2D87, 0xDB541CC6, + 0x94158A01, 0x8D0EBB40, 0xA623E883, 0xBF38D9C2, + 0x38A0C50D, 0x21BBF44C, 0x0A96A78F, 0x138D96CE, + 0x5CCC0009, 0x45D73148, 0x6EFA628B, 0x77E153CA, + 0xBABB5D54, 0xA3A06C15, 0x888D3FD6, 0x91960E97, + 0xDED79850, 0xC7CCA911, 0xECE1FAD2, 0xF5FACB93, + 0x7262D75C, 0x6B79E61D, 0x4054B5DE, 0x594F849F, + 0x160E1258, 0x0F152319, 0x243870DA, 0x3D23419B, + 0x65FD6BA7, 0x7CE65AE6, 0x57CB0925, 0x4ED03864, + 0x0191AEA3, 0x188A9FE2, 0x33A7CC21, 0x2ABCFD60, + 0xAD24E1AF, 0xB43FD0EE, 0x9F12832D, 0x8609B26C, + 0xC94824AB, 0xD05315EA, 0xFB7E4629, 0xE2657768, + 0x2F3F79F6, 0x362448B7, 0x1D091B74, 0x04122A35, + 0x4B53BCF2, 0x52488DB3, 0x7965DE70, 0x607EEF31, + 0xE7E6F3FE, 0xFEFDC2BF, 0xD5D0917C, 0xCCCBA03D, + 0x838A36FA, 0x9A9107BB, 0xB1BC5478, 0xA8A76539, + 0x3B83984B, 0x2298A90A, 0x09B5FAC9, 0x10AECB88, + 0x5FEF5D4F, 0x46F46C0E, 0x6DD93FCD, 0x74C20E8C, + 0xF35A1243, 0xEA412302, 0xC16C70C1, 0xD8774180, + 0x9736D747, 0x8E2DE606, 0xA500B5C5, 0xBC1B8484, + 0x71418A1A, 0x685ABB5B, 0x4377E898, 0x5A6CD9D9, + 0x152D4F1E, 0x0C367E5F, 0x271B2D9C, 0x3E001CDD, + 0xB9980012, 0xA0833153, 0x8BAE6290, 0x92B553D1, + 0xDDF4C516, 0xC4EFF457, 0xEFC2A794, 0xF6D996D5, + 0xAE07BCE9, 0xB71C8DA8, 0x9C31DE6B, 0x852AEF2A, + 0xCA6B79ED, 0xD37048AC, 0xF85D1B6F, 0xE1462A2E, + 0x66DE36E1, 0x7FC507A0, 0x54E85463, 0x4DF36522, + 0x02B2F3E5, 0x1BA9C2A4, 0x30849167, 0x299FA026, + 0xE4C5AEB8, 0xFDDE9FF9, 0xD6F3CC3A, 0xCFE8FD7B, + 0x80A96BBC, 0x99B25AFD, 0xB29F093E, 0xAB84387F, + 0x2C1C24B0, 0x350715F1, 0x1E2A4632, 0x07317773, + 0x4870E1B4, 0x516BD0F5, 0x7A468336, 0x635DB277, + 0xCBFAD74E, 0xD2E1E60F, 0xF9CCB5CC, 0xE0D7848D, + 0xAF96124A, 0xB68D230B, 0x9DA070C8, 0x84BB4189, + 0x03235D46, 0x1A386C07, 0x31153FC4, 0x280E0E85, + 0x674F9842, 0x7E54A903, 0x5579FAC0, 0x4C62CB81, + 0x8138C51F, 0x9823F45E, 0xB30EA79D, 0xAA1596DC, + 0xE554001B, 0xFC4F315A, 0xD7626299, 0xCE7953D8, + 0x49E14F17, 0x50FA7E56, 0x7BD72D95, 0x62CC1CD4, + 0x2D8D8A13, 0x3496BB52, 0x1FBBE891, 0x06A0D9D0, + 0x5E7EF3EC, 0x4765C2AD, 0x6C48916E, 0x7553A02F, + 0x3A1236E8, 0x230907A9, 0x0824546A, 0x113F652B, + 0x96A779E4, 0x8FBC48A5, 0xA4911B66, 0xBD8A2A27, + 0xF2CBBCE0, 0xEBD08DA1, 0xC0FDDE62, 0xD9E6EF23, + 0x14BCE1BD, 0x0DA7D0FC, 0x268A833F, 0x3F91B27E, + 0x70D024B9, 0x69CB15F8, 0x42E6463B, 0x5BFD777A, + 0xDC656BB5, 0xC57E5AF4, 0xEE530937, 0xF7483876, + 0xB809AEB1, 0xA1129FF0, 0x8A3FCC33, 0x9324FD72 +}; +const uint32_t CRC32_T8_2[] = { + 0x00000000, 0x01C26A37, 0x0384D46E, 0x0246BE59, + 0x0709A8DC, 0x06CBC2EB, 0x048D7CB2, 0x054F1685, + 0x0E1351B8, 0x0FD13B8F, 0x0D9785D6, 0x0C55EFE1, + 0x091AF964, 0x08D89353, 0x0A9E2D0A, 0x0B5C473D, + 0x1C26A370, 0x1DE4C947, 0x1FA2771E, 0x1E601D29, + 0x1B2F0BAC, 0x1AED619B, 0x18ABDFC2, 0x1969B5F5, + 0x1235F2C8, 0x13F798FF, 0x11B126A6, 0x10734C91, + 0x153C5A14, 0x14FE3023, 0x16B88E7A, 0x177AE44D, + 0x384D46E0, 0x398F2CD7, 0x3BC9928E, 0x3A0BF8B9, + 0x3F44EE3C, 0x3E86840B, 0x3CC03A52, 0x3D025065, + 0x365E1758, 0x379C7D6F, 0x35DAC336, 0x3418A901, + 0x3157BF84, 0x3095D5B3, 0x32D36BEA, 0x331101DD, + 0x246BE590, 0x25A98FA7, 0x27EF31FE, 0x262D5BC9, + 0x23624D4C, 0x22A0277B, 0x20E69922, 0x2124F315, + 0x2A78B428, 0x2BBADE1F, 0x29FC6046, 0x283E0A71, + 0x2D711CF4, 0x2CB376C3, 0x2EF5C89A, 0x2F37A2AD, + 0x709A8DC0, 0x7158E7F7, 0x731E59AE, 0x72DC3399, + 0x7793251C, 0x76514F2B, 0x7417F172, 0x75D59B45, + 0x7E89DC78, 0x7F4BB64F, 0x7D0D0816, 0x7CCF6221, + 0x798074A4, 0x78421E93, 0x7A04A0CA, 0x7BC6CAFD, + 0x6CBC2EB0, 0x6D7E4487, 0x6F38FADE, 0x6EFA90E9, + 0x6BB5866C, 0x6A77EC5B, 0x68315202, 0x69F33835, + 0x62AF7F08, 0x636D153F, 0x612BAB66, 0x60E9C151, + 0x65A6D7D4, 0x6464BDE3, 0x662203BA, 0x67E0698D, + 0x48D7CB20, 0x4915A117, 0x4B531F4E, 0x4A917579, + 0x4FDE63FC, 0x4E1C09CB, 0x4C5AB792, 0x4D98DDA5, + 0x46C49A98, 0x4706F0AF, 0x45404EF6, 0x448224C1, + 0x41CD3244, 0x400F5873, 0x4249E62A, 0x438B8C1D, + 0x54F16850, 0x55330267, 0x5775BC3E, 0x56B7D609, + 0x53F8C08C, 0x523AAABB, 0x507C14E2, 0x51BE7ED5, + 0x5AE239E8, 0x5B2053DF, 0x5966ED86, 0x58A487B1, + 0x5DEB9134, 0x5C29FB03, 0x5E6F455A, 0x5FAD2F6D, + 0xE1351B80, 0xE0F771B7, 0xE2B1CFEE, 0xE373A5D9, + 0xE63CB35C, 0xE7FED96B, 0xE5B86732, 0xE47A0D05, + 0xEF264A38, 0xEEE4200F, 0xECA29E56, 0xED60F461, + 0xE82FE2E4, 0xE9ED88D3, 0xEBAB368A, 0xEA695CBD, + 0xFD13B8F0, 0xFCD1D2C7, 0xFE976C9E, 0xFF5506A9, + 0xFA1A102C, 0xFBD87A1B, 0xF99EC442, 0xF85CAE75, + 0xF300E948, 0xF2C2837F, 0xF0843D26, 0xF1465711, + 0xF4094194, 0xF5CB2BA3, 0xF78D95FA, 0xF64FFFCD, + 0xD9785D60, 0xD8BA3757, 0xDAFC890E, 0xDB3EE339, + 0xDE71F5BC, 0xDFB39F8B, 0xDDF521D2, 0xDC374BE5, + 0xD76B0CD8, 0xD6A966EF, 0xD4EFD8B6, 0xD52DB281, + 0xD062A404, 0xD1A0CE33, 0xD3E6706A, 0xD2241A5D, + 0xC55EFE10, 0xC49C9427, 0xC6DA2A7E, 0xC7184049, + 0xC25756CC, 0xC3953CFB, 0xC1D382A2, 0xC011E895, + 0xCB4DAFA8, 0xCA8FC59F, 0xC8C97BC6, 0xC90B11F1, + 0xCC440774, 0xCD866D43, 0xCFC0D31A, 0xCE02B92D, + 0x91AF9640, 0x906DFC77, 0x922B422E, 0x93E92819, + 0x96A63E9C, 0x976454AB, 0x9522EAF2, 0x94E080C5, + 0x9FBCC7F8, 0x9E7EADCF, 0x9C381396, 0x9DFA79A1, + 0x98B56F24, 0x99770513, 0x9B31BB4A, 0x9AF3D17D, + 0x8D893530, 0x8C4B5F07, 0x8E0DE15E, 0x8FCF8B69, + 0x8A809DEC, 0x8B42F7DB, 0x89044982, 0x88C623B5, + 0x839A6488, 0x82580EBF, 0x801EB0E6, 0x81DCDAD1, + 0x8493CC54, 0x8551A663, 0x8717183A, 0x86D5720D, + 0xA9E2D0A0, 0xA820BA97, 0xAA6604CE, 0xABA46EF9, + 0xAEEB787C, 0xAF29124B, 0xAD6FAC12, 0xACADC625, + 0xA7F18118, 0xA633EB2F, 0xA4755576, 0xA5B73F41, + 0xA0F829C4, 0xA13A43F3, 0xA37CFDAA, 0xA2BE979D, + 0xB5C473D0, 0xB40619E7, 0xB640A7BE, 0xB782CD89, + 0xB2CDDB0C, 0xB30FB13B, 0xB1490F62, 0xB08B6555, + 0xBBD72268, 0xBA15485F, 0xB853F606, 0xB9919C31, + 0xBCDE8AB4, 0xBD1CE083, 0xBF5A5EDA, 0xBE9834ED +}; +const uint32_t CRC32_T8_3[] = { + 0x00000000, 0xB8BC6765, 0xAA09C88B, 0x12B5AFEE, + 0x8F629757, 0x37DEF032, 0x256B5FDC, 0x9DD738B9, + 0xC5B428EF, 0x7D084F8A, 0x6FBDE064, 0xD7018701, + 0x4AD6BFB8, 0xF26AD8DD, 0xE0DF7733, 0x58631056, + 0x5019579F, 0xE8A530FA, 0xFA109F14, 0x42ACF871, + 0xDF7BC0C8, 0x67C7A7AD, 0x75720843, 0xCDCE6F26, + 0x95AD7F70, 0x2D111815, 0x3FA4B7FB, 0x8718D09E, + 0x1ACFE827, 0xA2738F42, 0xB0C620AC, 0x087A47C9, + 0xA032AF3E, 0x188EC85B, 0x0A3B67B5, 0xB28700D0, + 0x2F503869, 0x97EC5F0C, 0x8559F0E2, 0x3DE59787, + 0x658687D1, 0xDD3AE0B4, 0xCF8F4F5A, 0x7733283F, + 0xEAE41086, 0x525877E3, 0x40EDD80D, 0xF851BF68, + 0xF02BF8A1, 0x48979FC4, 0x5A22302A, 0xE29E574F, + 0x7F496FF6, 0xC7F50893, 0xD540A77D, 0x6DFCC018, + 0x359FD04E, 0x8D23B72B, 0x9F9618C5, 0x272A7FA0, + 0xBAFD4719, 0x0241207C, 0x10F48F92, 0xA848E8F7, + 0x9B14583D, 0x23A83F58, 0x311D90B6, 0x89A1F7D3, + 0x1476CF6A, 0xACCAA80F, 0xBE7F07E1, 0x06C36084, + 0x5EA070D2, 0xE61C17B7, 0xF4A9B859, 0x4C15DF3C, + 0xD1C2E785, 0x697E80E0, 0x7BCB2F0E, 0xC377486B, + 0xCB0D0FA2, 0x73B168C7, 0x6104C729, 0xD9B8A04C, + 0x446F98F5, 0xFCD3FF90, 0xEE66507E, 0x56DA371B, + 0x0EB9274D, 0xB6054028, 0xA4B0EFC6, 0x1C0C88A3, + 0x81DBB01A, 0x3967D77F, 0x2BD27891, 0x936E1FF4, + 0x3B26F703, 0x839A9066, 0x912F3F88, 0x299358ED, + 0xB4446054, 0x0CF80731, 0x1E4DA8DF, 0xA6F1CFBA, + 0xFE92DFEC, 0x462EB889, 0x549B1767, 0xEC277002, + 0x71F048BB, 0xC94C2FDE, 0xDBF98030, 0x6345E755, + 0x6B3FA09C, 0xD383C7F9, 0xC1366817, 0x798A0F72, + 0xE45D37CB, 0x5CE150AE, 0x4E54FF40, 0xF6E89825, + 0xAE8B8873, 0x1637EF16, 0x048240F8, 0xBC3E279D, + 0x21E91F24, 0x99557841, 0x8BE0D7AF, 0x335CB0CA, + 0xED59B63B, 0x55E5D15E, 0x47507EB0, 0xFFEC19D5, + 0x623B216C, 0xDA874609, 0xC832E9E7, 0x708E8E82, + 0x28ED9ED4, 0x9051F9B1, 0x82E4565F, 0x3A58313A, + 0xA78F0983, 0x1F336EE6, 0x0D86C108, 0xB53AA66D, + 0xBD40E1A4, 0x05FC86C1, 0x1749292F, 0xAFF54E4A, + 0x322276F3, 0x8A9E1196, 0x982BBE78, 0x2097D91D, + 0x78F4C94B, 0xC048AE2E, 0xD2FD01C0, 0x6A4166A5, + 0xF7965E1C, 0x4F2A3979, 0x5D9F9697, 0xE523F1F2, + 0x4D6B1905, 0xF5D77E60, 0xE762D18E, 0x5FDEB6EB, + 0xC2098E52, 0x7AB5E937, 0x680046D9, 0xD0BC21BC, + 0x88DF31EA, 0x3063568F, 0x22D6F961, 0x9A6A9E04, + 0x07BDA6BD, 0xBF01C1D8, 0xADB46E36, 0x15080953, + 0x1D724E9A, 0xA5CE29FF, 0xB77B8611, 0x0FC7E174, + 0x9210D9CD, 0x2AACBEA8, 0x38191146, 0x80A57623, + 0xD8C66675, 0x607A0110, 0x72CFAEFE, 0xCA73C99B, + 0x57A4F122, 0xEF189647, 0xFDAD39A9, 0x45115ECC, + 0x764DEE06, 0xCEF18963, 0xDC44268D, 0x64F841E8, + 0xF92F7951, 0x41931E34, 0x5326B1DA, 0xEB9AD6BF, + 0xB3F9C6E9, 0x0B45A18C, 0x19F00E62, 0xA14C6907, + 0x3C9B51BE, 0x842736DB, 0x96929935, 0x2E2EFE50, + 0x2654B999, 0x9EE8DEFC, 0x8C5D7112, 0x34E11677, + 0xA9362ECE, 0x118A49AB, 0x033FE645, 0xBB838120, + 0xE3E09176, 0x5B5CF613, 0x49E959FD, 0xF1553E98, + 0x6C820621, 0xD43E6144, 0xC68BCEAA, 0x7E37A9CF, + 0xD67F4138, 0x6EC3265D, 0x7C7689B3, 0xC4CAEED6, + 0x591DD66F, 0xE1A1B10A, 0xF3141EE4, 0x4BA87981, + 0x13CB69D7, 0xAB770EB2, 0xB9C2A15C, 0x017EC639, + 0x9CA9FE80, 0x241599E5, 0x36A0360B, 0x8E1C516E, + 0x866616A7, 0x3EDA71C2, 0x2C6FDE2C, 0x94D3B949, + 0x090481F0, 0xB1B8E695, 0xA30D497B, 0x1BB12E1E, + 0x43D23E48, 0xFB6E592D, 0xE9DBF6C3, 0x516791A6, + 0xCCB0A91F, 0x740CCE7A, 0x66B96194, 0xDE0506F1 +}; +const uint32_t CRC32_T8_4[] = { + 0x00000000, 0x3D6029B0, 0x7AC05360, 0x47A07AD0, + 0xF580A6C0, 0xC8E08F70, 0x8F40F5A0, 0xB220DC10, + 0x30704BC1, 0x0D106271, 0x4AB018A1, 0x77D03111, + 0xC5F0ED01, 0xF890C4B1, 0xBF30BE61, 0x825097D1, + 0x60E09782, 0x5D80BE32, 0x1A20C4E2, 0x2740ED52, + 0x95603142, 0xA80018F2, 0xEFA06222, 0xD2C04B92, + 0x5090DC43, 0x6DF0F5F3, 0x2A508F23, 0x1730A693, + 0xA5107A83, 0x98705333, 0xDFD029E3, 0xE2B00053, + 0xC1C12F04, 0xFCA106B4, 0xBB017C64, 0x866155D4, + 0x344189C4, 0x0921A074, 0x4E81DAA4, 0x73E1F314, + 0xF1B164C5, 0xCCD14D75, 0x8B7137A5, 0xB6111E15, + 0x0431C205, 0x3951EBB5, 0x7EF19165, 0x4391B8D5, + 0xA121B886, 0x9C419136, 0xDBE1EBE6, 0xE681C256, + 0x54A11E46, 0x69C137F6, 0x2E614D26, 0x13016496, + 0x9151F347, 0xAC31DAF7, 0xEB91A027, 0xD6F18997, + 0x64D15587, 0x59B17C37, 0x1E1106E7, 0x23712F57, + 0x58F35849, 0x659371F9, 0x22330B29, 0x1F532299, + 0xAD73FE89, 0x9013D739, 0xD7B3ADE9, 0xEAD38459, + 0x68831388, 0x55E33A38, 0x124340E8, 0x2F236958, + 0x9D03B548, 0xA0639CF8, 0xE7C3E628, 0xDAA3CF98, + 0x3813CFCB, 0x0573E67B, 0x42D39CAB, 0x7FB3B51B, + 0xCD93690B, 0xF0F340BB, 0xB7533A6B, 0x8A3313DB, + 0x0863840A, 0x3503ADBA, 0x72A3D76A, 0x4FC3FEDA, + 0xFDE322CA, 0xC0830B7A, 0x872371AA, 0xBA43581A, + 0x9932774D, 0xA4525EFD, 0xE3F2242D, 0xDE920D9D, + 0x6CB2D18D, 0x51D2F83D, 0x167282ED, 0x2B12AB5D, + 0xA9423C8C, 0x9422153C, 0xD3826FEC, 0xEEE2465C, + 0x5CC29A4C, 0x61A2B3FC, 0x2602C92C, 0x1B62E09C, + 0xF9D2E0CF, 0xC4B2C97F, 0x8312B3AF, 0xBE729A1F, + 0x0C52460F, 0x31326FBF, 0x7692156F, 0x4BF23CDF, + 0xC9A2AB0E, 0xF4C282BE, 0xB362F86E, 0x8E02D1DE, + 0x3C220DCE, 0x0142247E, 0x46E25EAE, 0x7B82771E, + 0xB1E6B092, 0x8C869922, 0xCB26E3F2, 0xF646CA42, + 0x44661652, 0x79063FE2, 0x3EA64532, 0x03C66C82, + 0x8196FB53, 0xBCF6D2E3, 0xFB56A833, 0xC6368183, + 0x74165D93, 0x49767423, 0x0ED60EF3, 0x33B62743, + 0xD1062710, 0xEC660EA0, 0xABC67470, 0x96A65DC0, + 0x248681D0, 0x19E6A860, 0x5E46D2B0, 0x6326FB00, + 0xE1766CD1, 0xDC164561, 0x9BB63FB1, 0xA6D61601, + 0x14F6CA11, 0x2996E3A1, 0x6E369971, 0x5356B0C1, + 0x70279F96, 0x4D47B626, 0x0AE7CCF6, 0x3787E546, + 0x85A73956, 0xB8C710E6, 0xFF676A36, 0xC2074386, + 0x4057D457, 0x7D37FDE7, 0x3A978737, 0x07F7AE87, + 0xB5D77297, 0x88B75B27, 0xCF1721F7, 0xF2770847, + 0x10C70814, 0x2DA721A4, 0x6A075B74, 0x576772C4, + 0xE547AED4, 0xD8278764, 0x9F87FDB4, 0xA2E7D404, + 0x20B743D5, 0x1DD76A65, 0x5A7710B5, 0x67173905, + 0xD537E515, 0xE857CCA5, 0xAFF7B675, 0x92979FC5, + 0xE915E8DB, 0xD475C16B, 0x93D5BBBB, 0xAEB5920B, + 0x1C954E1B, 0x21F567AB, 0x66551D7B, 0x5B3534CB, + 0xD965A31A, 0xE4058AAA, 0xA3A5F07A, 0x9EC5D9CA, + 0x2CE505DA, 0x11852C6A, 0x562556BA, 0x6B457F0A, + 0x89F57F59, 0xB49556E9, 0xF3352C39, 0xCE550589, + 0x7C75D999, 0x4115F029, 0x06B58AF9, 0x3BD5A349, + 0xB9853498, 0x84E51D28, 0xC34567F8, 0xFE254E48, + 0x4C059258, 0x7165BBE8, 0x36C5C138, 0x0BA5E888, + 0x28D4C7DF, 0x15B4EE6F, 0x521494BF, 0x6F74BD0F, + 0xDD54611F, 0xE03448AF, 0xA794327F, 0x9AF41BCF, + 0x18A48C1E, 0x25C4A5AE, 0x6264DF7E, 0x5F04F6CE, + 0xED242ADE, 0xD044036E, 0x97E479BE, 0xAA84500E, + 0x4834505D, 0x755479ED, 0x32F4033D, 0x0F942A8D, + 0xBDB4F69D, 0x80D4DF2D, 0xC774A5FD, 0xFA148C4D, + 0x78441B9C, 0x4524322C, 0x028448FC, 0x3FE4614C, + 0x8DC4BD5C, 0xB0A494EC, 0xF704EE3C, 0xCA64C78C +}; +const uint32_t CRC32_T8_5[] = { + 0x00000000, 0xCB5CD3A5, 0x4DC8A10B, 0x869472AE, + 0x9B914216, 0x50CD91B3, 0xD659E31D, 0x1D0530B8, + 0xEC53826D, 0x270F51C8, 0xA19B2366, 0x6AC7F0C3, + 0x77C2C07B, 0xBC9E13DE, 0x3A0A6170, 0xF156B2D5, + 0x03D6029B, 0xC88AD13E, 0x4E1EA390, 0x85427035, + 0x9847408D, 0x531B9328, 0xD58FE186, 0x1ED33223, + 0xEF8580F6, 0x24D95353, 0xA24D21FD, 0x6911F258, + 0x7414C2E0, 0xBF481145, 0x39DC63EB, 0xF280B04E, + 0x07AC0536, 0xCCF0D693, 0x4A64A43D, 0x81387798, + 0x9C3D4720, 0x57619485, 0xD1F5E62B, 0x1AA9358E, + 0xEBFF875B, 0x20A354FE, 0xA6372650, 0x6D6BF5F5, + 0x706EC54D, 0xBB3216E8, 0x3DA66446, 0xF6FAB7E3, + 0x047A07AD, 0xCF26D408, 0x49B2A6A6, 0x82EE7503, + 0x9FEB45BB, 0x54B7961E, 0xD223E4B0, 0x197F3715, + 0xE82985C0, 0x23755665, 0xA5E124CB, 0x6EBDF76E, + 0x73B8C7D6, 0xB8E41473, 0x3E7066DD, 0xF52CB578, + 0x0F580A6C, 0xC404D9C9, 0x4290AB67, 0x89CC78C2, + 0x94C9487A, 0x5F959BDF, 0xD901E971, 0x125D3AD4, + 0xE30B8801, 0x28575BA4, 0xAEC3290A, 0x659FFAAF, + 0x789ACA17, 0xB3C619B2, 0x35526B1C, 0xFE0EB8B9, + 0x0C8E08F7, 0xC7D2DB52, 0x4146A9FC, 0x8A1A7A59, + 0x971F4AE1, 0x5C439944, 0xDAD7EBEA, 0x118B384F, + 0xE0DD8A9A, 0x2B81593F, 0xAD152B91, 0x6649F834, + 0x7B4CC88C, 0xB0101B29, 0x36846987, 0xFDD8BA22, + 0x08F40F5A, 0xC3A8DCFF, 0x453CAE51, 0x8E607DF4, + 0x93654D4C, 0x58399EE9, 0xDEADEC47, 0x15F13FE2, + 0xE4A78D37, 0x2FFB5E92, 0xA96F2C3C, 0x6233FF99, + 0x7F36CF21, 0xB46A1C84, 0x32FE6E2A, 0xF9A2BD8F, + 0x0B220DC1, 0xC07EDE64, 0x46EAACCA, 0x8DB67F6F, + 0x90B34FD7, 0x5BEF9C72, 0xDD7BEEDC, 0x16273D79, + 0xE7718FAC, 0x2C2D5C09, 0xAAB92EA7, 0x61E5FD02, + 0x7CE0CDBA, 0xB7BC1E1F, 0x31286CB1, 0xFA74BF14, + 0x1EB014D8, 0xD5ECC77D, 0x5378B5D3, 0x98246676, + 0x852156CE, 0x4E7D856B, 0xC8E9F7C5, 0x03B52460, + 0xF2E396B5, 0x39BF4510, 0xBF2B37BE, 0x7477E41B, + 0x6972D4A3, 0xA22E0706, 0x24BA75A8, 0xEFE6A60D, + 0x1D661643, 0xD63AC5E6, 0x50AEB748, 0x9BF264ED, + 0x86F75455, 0x4DAB87F0, 0xCB3FF55E, 0x006326FB, + 0xF135942E, 0x3A69478B, 0xBCFD3525, 0x77A1E680, + 0x6AA4D638, 0xA1F8059D, 0x276C7733, 0xEC30A496, + 0x191C11EE, 0xD240C24B, 0x54D4B0E5, 0x9F886340, + 0x828D53F8, 0x49D1805D, 0xCF45F2F3, 0x04192156, + 0xF54F9383, 0x3E134026, 0xB8873288, 0x73DBE12D, + 0x6EDED195, 0xA5820230, 0x2316709E, 0xE84AA33B, + 0x1ACA1375, 0xD196C0D0, 0x5702B27E, 0x9C5E61DB, + 0x815B5163, 0x4A0782C6, 0xCC93F068, 0x07CF23CD, + 0xF6999118, 0x3DC542BD, 0xBB513013, 0x700DE3B6, + 0x6D08D30E, 0xA65400AB, 0x20C07205, 0xEB9CA1A0, + 0x11E81EB4, 0xDAB4CD11, 0x5C20BFBF, 0x977C6C1A, + 0x8A795CA2, 0x41258F07, 0xC7B1FDA9, 0x0CED2E0C, + 0xFDBB9CD9, 0x36E74F7C, 0xB0733DD2, 0x7B2FEE77, + 0x662ADECF, 0xAD760D6A, 0x2BE27FC4, 0xE0BEAC61, + 0x123E1C2F, 0xD962CF8A, 0x5FF6BD24, 0x94AA6E81, + 0x89AF5E39, 0x42F38D9C, 0xC467FF32, 0x0F3B2C97, + 0xFE6D9E42, 0x35314DE7, 0xB3A53F49, 0x78F9ECEC, + 0x65FCDC54, 0xAEA00FF1, 0x28347D5F, 0xE368AEFA, + 0x16441B82, 0xDD18C827, 0x5B8CBA89, 0x90D0692C, + 0x8DD55994, 0x46898A31, 0xC01DF89F, 0x0B412B3A, + 0xFA1799EF, 0x314B4A4A, 0xB7DF38E4, 0x7C83EB41, + 0x6186DBF9, 0xAADA085C, 0x2C4E7AF2, 0xE712A957, + 0x15921919, 0xDECECABC, 0x585AB812, 0x93066BB7, + 0x8E035B0F, 0x455F88AA, 0xC3CBFA04, 0x089729A1, + 0xF9C19B74, 0x329D48D1, 0xB4093A7F, 0x7F55E9DA, + 0x6250D962, 0xA90C0AC7, 0x2F987869, 0xE4C4ABCC +}; +const uint32_t CRC32_T8_6[] = { + 0x00000000, 0xA6770BB4, 0x979F1129, 0x31E81A9D, + 0xF44F2413, 0x52382FA7, 0x63D0353A, 0xC5A73E8E, + 0x33EF4E67, 0x959845D3, 0xA4705F4E, 0x020754FA, + 0xC7A06A74, 0x61D761C0, 0x503F7B5D, 0xF64870E9, + 0x67DE9CCE, 0xC1A9977A, 0xF0418DE7, 0x56368653, + 0x9391B8DD, 0x35E6B369, 0x040EA9F4, 0xA279A240, + 0x5431D2A9, 0xF246D91D, 0xC3AEC380, 0x65D9C834, + 0xA07EF6BA, 0x0609FD0E, 0x37E1E793, 0x9196EC27, + 0xCFBD399C, 0x69CA3228, 0x582228B5, 0xFE552301, + 0x3BF21D8F, 0x9D85163B, 0xAC6D0CA6, 0x0A1A0712, + 0xFC5277FB, 0x5A257C4F, 0x6BCD66D2, 0xCDBA6D66, + 0x081D53E8, 0xAE6A585C, 0x9F8242C1, 0x39F54975, + 0xA863A552, 0x0E14AEE6, 0x3FFCB47B, 0x998BBFCF, + 0x5C2C8141, 0xFA5B8AF5, 0xCBB39068, 0x6DC49BDC, + 0x9B8CEB35, 0x3DFBE081, 0x0C13FA1C, 0xAA64F1A8, + 0x6FC3CF26, 0xC9B4C492, 0xF85CDE0F, 0x5E2BD5BB, + 0x440B7579, 0xE27C7ECD, 0xD3946450, 0x75E36FE4, + 0xB044516A, 0x16335ADE, 0x27DB4043, 0x81AC4BF7, + 0x77E43B1E, 0xD19330AA, 0xE07B2A37, 0x460C2183, + 0x83AB1F0D, 0x25DC14B9, 0x14340E24, 0xB2430590, + 0x23D5E9B7, 0x85A2E203, 0xB44AF89E, 0x123DF32A, + 0xD79ACDA4, 0x71EDC610, 0x4005DC8D, 0xE672D739, + 0x103AA7D0, 0xB64DAC64, 0x87A5B6F9, 0x21D2BD4D, + 0xE47583C3, 0x42028877, 0x73EA92EA, 0xD59D995E, + 0x8BB64CE5, 0x2DC14751, 0x1C295DCC, 0xBA5E5678, + 0x7FF968F6, 0xD98E6342, 0xE86679DF, 0x4E11726B, + 0xB8590282, 0x1E2E0936, 0x2FC613AB, 0x89B1181F, + 0x4C162691, 0xEA612D25, 0xDB8937B8, 0x7DFE3C0C, + 0xEC68D02B, 0x4A1FDB9F, 0x7BF7C102, 0xDD80CAB6, + 0x1827F438, 0xBE50FF8C, 0x8FB8E511, 0x29CFEEA5, + 0xDF879E4C, 0x79F095F8, 0x48188F65, 0xEE6F84D1, + 0x2BC8BA5F, 0x8DBFB1EB, 0xBC57AB76, 0x1A20A0C2, + 0x8816EAF2, 0x2E61E146, 0x1F89FBDB, 0xB9FEF06F, + 0x7C59CEE1, 0xDA2EC555, 0xEBC6DFC8, 0x4DB1D47C, + 0xBBF9A495, 0x1D8EAF21, 0x2C66B5BC, 0x8A11BE08, + 0x4FB68086, 0xE9C18B32, 0xD82991AF, 0x7E5E9A1B, + 0xEFC8763C, 0x49BF7D88, 0x78576715, 0xDE206CA1, + 0x1B87522F, 0xBDF0599B, 0x8C184306, 0x2A6F48B2, + 0xDC27385B, 0x7A5033EF, 0x4BB82972, 0xEDCF22C6, + 0x28681C48, 0x8E1F17FC, 0xBFF70D61, 0x198006D5, + 0x47ABD36E, 0xE1DCD8DA, 0xD034C247, 0x7643C9F3, + 0xB3E4F77D, 0x1593FCC9, 0x247BE654, 0x820CEDE0, + 0x74449D09, 0xD23396BD, 0xE3DB8C20, 0x45AC8794, + 0x800BB91A, 0x267CB2AE, 0x1794A833, 0xB1E3A387, + 0x20754FA0, 0x86024414, 0xB7EA5E89, 0x119D553D, + 0xD43A6BB3, 0x724D6007, 0x43A57A9A, 0xE5D2712E, + 0x139A01C7, 0xB5ED0A73, 0x840510EE, 0x22721B5A, + 0xE7D525D4, 0x41A22E60, 0x704A34FD, 0xD63D3F49, + 0xCC1D9F8B, 0x6A6A943F, 0x5B828EA2, 0xFDF58516, + 0x3852BB98, 0x9E25B02C, 0xAFCDAAB1, 0x09BAA105, + 0xFFF2D1EC, 0x5985DA58, 0x686DC0C5, 0xCE1ACB71, + 0x0BBDF5FF, 0xADCAFE4B, 0x9C22E4D6, 0x3A55EF62, + 0xABC30345, 0x0DB408F1, 0x3C5C126C, 0x9A2B19D8, + 0x5F8C2756, 0xF9FB2CE2, 0xC813367F, 0x6E643DCB, + 0x982C4D22, 0x3E5B4696, 0x0FB35C0B, 0xA9C457BF, + 0x6C636931, 0xCA146285, 0xFBFC7818, 0x5D8B73AC, + 0x03A0A617, 0xA5D7ADA3, 0x943FB73E, 0x3248BC8A, + 0xF7EF8204, 0x519889B0, 0x6070932D, 0xC6079899, + 0x304FE870, 0x9638E3C4, 0xA7D0F959, 0x01A7F2ED, + 0xC400CC63, 0x6277C7D7, 0x539FDD4A, 0xF5E8D6FE, + 0x647E3AD9, 0xC209316D, 0xF3E12BF0, 0x55962044, + 0x90311ECA, 0x3646157E, 0x07AE0FE3, 0xA1D90457, + 0x579174BE, 0xF1E67F0A, 0xC00E6597, 0x66796E23, + 0xA3DE50AD, 0x05A95B19, 0x34414184, 0x92364A30 +}; +const uint32_t CRC32_T8_7[] = { + 0x00000000, 0xCCAA009E, 0x4225077D, 0x8E8F07E3, + 0x844A0EFA, 0x48E00E64, 0xC66F0987, 0x0AC50919, + 0xD3E51BB5, 0x1F4F1B2B, 0x91C01CC8, 0x5D6A1C56, + 0x57AF154F, 0x9B0515D1, 0x158A1232, 0xD92012AC, + 0x7CBB312B, 0xB01131B5, 0x3E9E3656, 0xF23436C8, + 0xF8F13FD1, 0x345B3F4F, 0xBAD438AC, 0x767E3832, + 0xAF5E2A9E, 0x63F42A00, 0xED7B2DE3, 0x21D12D7D, + 0x2B142464, 0xE7BE24FA, 0x69312319, 0xA59B2387, + 0xF9766256, 0x35DC62C8, 0xBB53652B, 0x77F965B5, + 0x7D3C6CAC, 0xB1966C32, 0x3F196BD1, 0xF3B36B4F, + 0x2A9379E3, 0xE639797D, 0x68B67E9E, 0xA41C7E00, + 0xAED97719, 0x62737787, 0xECFC7064, 0x205670FA, + 0x85CD537D, 0x496753E3, 0xC7E85400, 0x0B42549E, + 0x01875D87, 0xCD2D5D19, 0x43A25AFA, 0x8F085A64, + 0x562848C8, 0x9A824856, 0x140D4FB5, 0xD8A74F2B, + 0xD2624632, 0x1EC846AC, 0x9047414F, 0x5CED41D1, + 0x299DC2ED, 0xE537C273, 0x6BB8C590, 0xA712C50E, + 0xADD7CC17, 0x617DCC89, 0xEFF2CB6A, 0x2358CBF4, + 0xFA78D958, 0x36D2D9C6, 0xB85DDE25, 0x74F7DEBB, + 0x7E32D7A2, 0xB298D73C, 0x3C17D0DF, 0xF0BDD041, + 0x5526F3C6, 0x998CF358, 0x1703F4BB, 0xDBA9F425, + 0xD16CFD3C, 0x1DC6FDA2, 0x9349FA41, 0x5FE3FADF, + 0x86C3E873, 0x4A69E8ED, 0xC4E6EF0E, 0x084CEF90, + 0x0289E689, 0xCE23E617, 0x40ACE1F4, 0x8C06E16A, + 0xD0EBA0BB, 0x1C41A025, 0x92CEA7C6, 0x5E64A758, + 0x54A1AE41, 0x980BAEDF, 0x1684A93C, 0xDA2EA9A2, + 0x030EBB0E, 0xCFA4BB90, 0x412BBC73, 0x8D81BCED, + 0x8744B5F4, 0x4BEEB56A, 0xC561B289, 0x09CBB217, + 0xAC509190, 0x60FA910E, 0xEE7596ED, 0x22DF9673, + 0x281A9F6A, 0xE4B09FF4, 0x6A3F9817, 0xA6959889, + 0x7FB58A25, 0xB31F8ABB, 0x3D908D58, 0xF13A8DC6, + 0xFBFF84DF, 0x37558441, 0xB9DA83A2, 0x7570833C, + 0x533B85DA, 0x9F918544, 0x111E82A7, 0xDDB48239, + 0xD7718B20, 0x1BDB8BBE, 0x95548C5D, 0x59FE8CC3, + 0x80DE9E6F, 0x4C749EF1, 0xC2FB9912, 0x0E51998C, + 0x04949095, 0xC83E900B, 0x46B197E8, 0x8A1B9776, + 0x2F80B4F1, 0xE32AB46F, 0x6DA5B38C, 0xA10FB312, + 0xABCABA0B, 0x6760BA95, 0xE9EFBD76, 0x2545BDE8, + 0xFC65AF44, 0x30CFAFDA, 0xBE40A839, 0x72EAA8A7, + 0x782FA1BE, 0xB485A120, 0x3A0AA6C3, 0xF6A0A65D, + 0xAA4DE78C, 0x66E7E712, 0xE868E0F1, 0x24C2E06F, + 0x2E07E976, 0xE2ADE9E8, 0x6C22EE0B, 0xA088EE95, + 0x79A8FC39, 0xB502FCA7, 0x3B8DFB44, 0xF727FBDA, + 0xFDE2F2C3, 0x3148F25D, 0xBFC7F5BE, 0x736DF520, + 0xD6F6D6A7, 0x1A5CD639, 0x94D3D1DA, 0x5879D144, + 0x52BCD85D, 0x9E16D8C3, 0x1099DF20, 0xDC33DFBE, + 0x0513CD12, 0xC9B9CD8C, 0x4736CA6F, 0x8B9CCAF1, + 0x8159C3E8, 0x4DF3C376, 0xC37CC495, 0x0FD6C40B, + 0x7AA64737, 0xB60C47A9, 0x3883404A, 0xF42940D4, + 0xFEEC49CD, 0x32464953, 0xBCC94EB0, 0x70634E2E, + 0xA9435C82, 0x65E95C1C, 0xEB665BFF, 0x27CC5B61, + 0x2D095278, 0xE1A352E6, 0x6F2C5505, 0xA386559B, + 0x061D761C, 0xCAB77682, 0x44387161, 0x889271FF, + 0x825778E6, 0x4EFD7878, 0xC0727F9B, 0x0CD87F05, + 0xD5F86DA9, 0x19526D37, 0x97DD6AD4, 0x5B776A4A, + 0x51B26353, 0x9D1863CD, 0x1397642E, 0xDF3D64B0, + 0x83D02561, 0x4F7A25FF, 0xC1F5221C, 0x0D5F2282, + 0x079A2B9B, 0xCB302B05, 0x45BF2CE6, 0x89152C78, + 0x50353ED4, 0x9C9F3E4A, 0x121039A9, 0xDEBA3937, + 0xD47F302E, 0x18D530B0, 0x965A3753, 0x5AF037CD, + 0xFF6B144A, 0x33C114D4, 0xBD4E1337, 0x71E413A9, + 0x7B211AB0, 0xB78B1A2E, 0x39041DCD, 0xF5AE1D53, + 0x2C8E0FFF, 0xE0240F61, 0x6EAB0882, 0xA201081C, + 0xA8C40105, 0x646E019B, 0xEAE10678, 0x264B06E6 +}; + + diff --git a/hadoop-common/src/main/native/src/org/apache/hadoop/util/crc32c_tables.h b/hadoop-common/src/main/native/src/org/apache/hadoop/util/crc32c_tables.h new file mode 100644 index 00000000000..d54c4dbefc8 --- /dev/null +++ b/hadoop-common/src/main/native/src/org/apache/hadoop/util/crc32c_tables.h @@ -0,0 +1,550 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * CRC-32 lookup tables generated by the polynomial 82F63B78 + * See also TestPureJavaCrc32.Table. + */ +const uint32_t CRC32C_T8_0[256] = { + 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4, + 0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB, + 0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B, + 0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24, + 0x105EC76F, 0xE235446C, 0xF165B798, 0x030E349B, + 0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384, + 0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, 0x89D76C54, + 0x5D1D08BF, 0xAF768BBC, 0xBC267848, 0x4E4DFB4B, + 0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A, + 0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35, + 0xAA64D611, 0x580F5512, 0x4B5FA6E6, 0xB93425E5, + 0x6DFE410E, 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA, + 0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45, + 0xF779DEAE, 0x05125DAD, 0x1642AE59, 0xE4292D5A, + 0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A, + 0x7DA08661, 0x8FCB0562, 0x9C9BF696, 0x6EF07595, + 0x417B1DBC, 0xB3109EBF, 0xA0406D4B, 0x522BEE48, + 0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957, + 0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687, + 0x0C38D26C, 0xFE53516F, 0xED03A29B, 0x1F682198, + 0x5125DAD3, 0xA34E59D0, 0xB01EAA24, 0x42752927, + 0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38, + 0xDBFC821C, 0x2997011F, 0x3AC7F2EB, 0xC8AC71E8, + 0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7, + 0x61C69362, 0x93AD1061, 0x80FDE395, 0x72966096, + 0xA65C047D, 0x5437877E, 0x4767748A, 0xB50CF789, + 0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859, + 0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46, + 0x7198540D, 0x83F3D70E, 0x90A324FA, 0x62C8A7F9, + 0xB602C312, 0x44694011, 0x5739B3E5, 0xA55230E6, + 0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36, + 0x3CDB9BDD, 0xCEB018DE, 0xDDE0EB2A, 0x2F8B6829, + 0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C, + 0x456CAC67, 0xB7072F64, 0xA457DC90, 0x563C5F93, + 0x082F63B7, 0xFA44E0B4, 0xE9141340, 0x1B7F9043, + 0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C, + 0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3, + 0x55326B08, 0xA759E80B, 0xB4091BFF, 0x466298FC, + 0x1871A4D8, 0xEA1A27DB, 0xF94AD42F, 0x0B21572C, + 0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033, + 0xA24BB5A6, 0x502036A5, 0x4370C551, 0xB11B4652, + 0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D, + 0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, 0x3BC21E9D, + 0xEF087A76, 0x1D63F975, 0x0E330A81, 0xFC588982, + 0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D, + 0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622, + 0x38CC2A06, 0xCAA7A905, 0xD9F75AF1, 0x2B9CD9F2, + 0xFF56BD19, 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED, + 0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530, + 0x0417B1DB, 0xF67C32D8, 0xE52CC12C, 0x1747422F, + 0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF, + 0x8ECEE914, 0x7CA56A17, 0x6FF599E3, 0x9D9E1AE0, + 0xD3D3E1AB, 0x21B862A8, 0x32E8915C, 0xC083125F, + 0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540, + 0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90, + 0x9E902E7B, 0x6CFBAD78, 0x7FAB5E8C, 0x8DC0DD8F, + 0xE330A81A, 0x115B2B19, 0x020BD8ED, 0xF0605BEE, + 0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1, + 0x69E9F0D5, 0x9B8273D6, 0x88D28022, 0x7AB90321, + 0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E, + 0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81, + 0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E, + 0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E, + 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351 +}; +const uint32_t CRC32C_T8_1[256] = { + 0x00000000, 0x13A29877, 0x274530EE, 0x34E7A899, + 0x4E8A61DC, 0x5D28F9AB, 0x69CF5132, 0x7A6DC945, + 0x9D14C3B8, 0x8EB65BCF, 0xBA51F356, 0xA9F36B21, + 0xD39EA264, 0xC03C3A13, 0xF4DB928A, 0xE7790AFD, + 0x3FC5F181, 0x2C6769F6, 0x1880C16F, 0x0B225918, + 0x714F905D, 0x62ED082A, 0x560AA0B3, 0x45A838C4, + 0xA2D13239, 0xB173AA4E, 0x859402D7, 0x96369AA0, + 0xEC5B53E5, 0xFFF9CB92, 0xCB1E630B, 0xD8BCFB7C, + 0x7F8BE302, 0x6C297B75, 0x58CED3EC, 0x4B6C4B9B, + 0x310182DE, 0x22A31AA9, 0x1644B230, 0x05E62A47, + 0xE29F20BA, 0xF13DB8CD, 0xC5DA1054, 0xD6788823, + 0xAC154166, 0xBFB7D911, 0x8B507188, 0x98F2E9FF, + 0x404E1283, 0x53EC8AF4, 0x670B226D, 0x74A9BA1A, + 0x0EC4735F, 0x1D66EB28, 0x298143B1, 0x3A23DBC6, + 0xDD5AD13B, 0xCEF8494C, 0xFA1FE1D5, 0xE9BD79A2, + 0x93D0B0E7, 0x80722890, 0xB4958009, 0xA737187E, + 0xFF17C604, 0xECB55E73, 0xD852F6EA, 0xCBF06E9D, + 0xB19DA7D8, 0xA23F3FAF, 0x96D89736, 0x857A0F41, + 0x620305BC, 0x71A19DCB, 0x45463552, 0x56E4AD25, + 0x2C896460, 0x3F2BFC17, 0x0BCC548E, 0x186ECCF9, + 0xC0D23785, 0xD370AFF2, 0xE797076B, 0xF4359F1C, + 0x8E585659, 0x9DFACE2E, 0xA91D66B7, 0xBABFFEC0, + 0x5DC6F43D, 0x4E646C4A, 0x7A83C4D3, 0x69215CA4, + 0x134C95E1, 0x00EE0D96, 0x3409A50F, 0x27AB3D78, + 0x809C2506, 0x933EBD71, 0xA7D915E8, 0xB47B8D9F, + 0xCE1644DA, 0xDDB4DCAD, 0xE9537434, 0xFAF1EC43, + 0x1D88E6BE, 0x0E2A7EC9, 0x3ACDD650, 0x296F4E27, + 0x53028762, 0x40A01F15, 0x7447B78C, 0x67E52FFB, + 0xBF59D487, 0xACFB4CF0, 0x981CE469, 0x8BBE7C1E, + 0xF1D3B55B, 0xE2712D2C, 0xD69685B5, 0xC5341DC2, + 0x224D173F, 0x31EF8F48, 0x050827D1, 0x16AABFA6, + 0x6CC776E3, 0x7F65EE94, 0x4B82460D, 0x5820DE7A, + 0xFBC3FAF9, 0xE861628E, 0xDC86CA17, 0xCF245260, + 0xB5499B25, 0xA6EB0352, 0x920CABCB, 0x81AE33BC, + 0x66D73941, 0x7575A136, 0x419209AF, 0x523091D8, + 0x285D589D, 0x3BFFC0EA, 0x0F186873, 0x1CBAF004, + 0xC4060B78, 0xD7A4930F, 0xE3433B96, 0xF0E1A3E1, + 0x8A8C6AA4, 0x992EF2D3, 0xADC95A4A, 0xBE6BC23D, + 0x5912C8C0, 0x4AB050B7, 0x7E57F82E, 0x6DF56059, + 0x1798A91C, 0x043A316B, 0x30DD99F2, 0x237F0185, + 0x844819FB, 0x97EA818C, 0xA30D2915, 0xB0AFB162, + 0xCAC27827, 0xD960E050, 0xED8748C9, 0xFE25D0BE, + 0x195CDA43, 0x0AFE4234, 0x3E19EAAD, 0x2DBB72DA, + 0x57D6BB9F, 0x447423E8, 0x70938B71, 0x63311306, + 0xBB8DE87A, 0xA82F700D, 0x9CC8D894, 0x8F6A40E3, + 0xF50789A6, 0xE6A511D1, 0xD242B948, 0xC1E0213F, + 0x26992BC2, 0x353BB3B5, 0x01DC1B2C, 0x127E835B, + 0x68134A1E, 0x7BB1D269, 0x4F567AF0, 0x5CF4E287, + 0x04D43CFD, 0x1776A48A, 0x23910C13, 0x30339464, + 0x4A5E5D21, 0x59FCC556, 0x6D1B6DCF, 0x7EB9F5B8, + 0x99C0FF45, 0x8A626732, 0xBE85CFAB, 0xAD2757DC, + 0xD74A9E99, 0xC4E806EE, 0xF00FAE77, 0xE3AD3600, + 0x3B11CD7C, 0x28B3550B, 0x1C54FD92, 0x0FF665E5, + 0x759BACA0, 0x663934D7, 0x52DE9C4E, 0x417C0439, + 0xA6050EC4, 0xB5A796B3, 0x81403E2A, 0x92E2A65D, + 0xE88F6F18, 0xFB2DF76F, 0xCFCA5FF6, 0xDC68C781, + 0x7B5FDFFF, 0x68FD4788, 0x5C1AEF11, 0x4FB87766, + 0x35D5BE23, 0x26772654, 0x12908ECD, 0x013216BA, + 0xE64B1C47, 0xF5E98430, 0xC10E2CA9, 0xD2ACB4DE, + 0xA8C17D9B, 0xBB63E5EC, 0x8F844D75, 0x9C26D502, + 0x449A2E7E, 0x5738B609, 0x63DF1E90, 0x707D86E7, + 0x0A104FA2, 0x19B2D7D5, 0x2D557F4C, 0x3EF7E73B, + 0xD98EEDC6, 0xCA2C75B1, 0xFECBDD28, 0xED69455F, + 0x97048C1A, 0x84A6146D, 0xB041BCF4, 0xA3E32483 +}; +const uint32_t CRC32C_T8_2[256] = { + 0x00000000, 0xA541927E, 0x4F6F520D, 0xEA2EC073, + 0x9EDEA41A, 0x3B9F3664, 0xD1B1F617, 0x74F06469, + 0x38513EC5, 0x9D10ACBB, 0x773E6CC8, 0xD27FFEB6, + 0xA68F9ADF, 0x03CE08A1, 0xE9E0C8D2, 0x4CA15AAC, + 0x70A27D8A, 0xD5E3EFF4, 0x3FCD2F87, 0x9A8CBDF9, + 0xEE7CD990, 0x4B3D4BEE, 0xA1138B9D, 0x045219E3, + 0x48F3434F, 0xEDB2D131, 0x079C1142, 0xA2DD833C, + 0xD62DE755, 0x736C752B, 0x9942B558, 0x3C032726, + 0xE144FB14, 0x4405696A, 0xAE2BA919, 0x0B6A3B67, + 0x7F9A5F0E, 0xDADBCD70, 0x30F50D03, 0x95B49F7D, + 0xD915C5D1, 0x7C5457AF, 0x967A97DC, 0x333B05A2, + 0x47CB61CB, 0xE28AF3B5, 0x08A433C6, 0xADE5A1B8, + 0x91E6869E, 0x34A714E0, 0xDE89D493, 0x7BC846ED, + 0x0F382284, 0xAA79B0FA, 0x40577089, 0xE516E2F7, + 0xA9B7B85B, 0x0CF62A25, 0xE6D8EA56, 0x43997828, + 0x37691C41, 0x92288E3F, 0x78064E4C, 0xDD47DC32, + 0xC76580D9, 0x622412A7, 0x880AD2D4, 0x2D4B40AA, + 0x59BB24C3, 0xFCFAB6BD, 0x16D476CE, 0xB395E4B0, + 0xFF34BE1C, 0x5A752C62, 0xB05BEC11, 0x151A7E6F, + 0x61EA1A06, 0xC4AB8878, 0x2E85480B, 0x8BC4DA75, + 0xB7C7FD53, 0x12866F2D, 0xF8A8AF5E, 0x5DE93D20, + 0x29195949, 0x8C58CB37, 0x66760B44, 0xC337993A, + 0x8F96C396, 0x2AD751E8, 0xC0F9919B, 0x65B803E5, + 0x1148678C, 0xB409F5F2, 0x5E273581, 0xFB66A7FF, + 0x26217BCD, 0x8360E9B3, 0x694E29C0, 0xCC0FBBBE, + 0xB8FFDFD7, 0x1DBE4DA9, 0xF7908DDA, 0x52D11FA4, + 0x1E704508, 0xBB31D776, 0x511F1705, 0xF45E857B, + 0x80AEE112, 0x25EF736C, 0xCFC1B31F, 0x6A802161, + 0x56830647, 0xF3C29439, 0x19EC544A, 0xBCADC634, + 0xC85DA25D, 0x6D1C3023, 0x8732F050, 0x2273622E, + 0x6ED23882, 0xCB93AAFC, 0x21BD6A8F, 0x84FCF8F1, + 0xF00C9C98, 0x554D0EE6, 0xBF63CE95, 0x1A225CEB, + 0x8B277743, 0x2E66E53D, 0xC448254E, 0x6109B730, + 0x15F9D359, 0xB0B84127, 0x5A968154, 0xFFD7132A, + 0xB3764986, 0x1637DBF8, 0xFC191B8B, 0x595889F5, + 0x2DA8ED9C, 0x88E97FE2, 0x62C7BF91, 0xC7862DEF, + 0xFB850AC9, 0x5EC498B7, 0xB4EA58C4, 0x11ABCABA, + 0x655BAED3, 0xC01A3CAD, 0x2A34FCDE, 0x8F756EA0, + 0xC3D4340C, 0x6695A672, 0x8CBB6601, 0x29FAF47F, + 0x5D0A9016, 0xF84B0268, 0x1265C21B, 0xB7245065, + 0x6A638C57, 0xCF221E29, 0x250CDE5A, 0x804D4C24, + 0xF4BD284D, 0x51FCBA33, 0xBBD27A40, 0x1E93E83E, + 0x5232B292, 0xF77320EC, 0x1D5DE09F, 0xB81C72E1, + 0xCCEC1688, 0x69AD84F6, 0x83834485, 0x26C2D6FB, + 0x1AC1F1DD, 0xBF8063A3, 0x55AEA3D0, 0xF0EF31AE, + 0x841F55C7, 0x215EC7B9, 0xCB7007CA, 0x6E3195B4, + 0x2290CF18, 0x87D15D66, 0x6DFF9D15, 0xC8BE0F6B, + 0xBC4E6B02, 0x190FF97C, 0xF321390F, 0x5660AB71, + 0x4C42F79A, 0xE90365E4, 0x032DA597, 0xA66C37E9, + 0xD29C5380, 0x77DDC1FE, 0x9DF3018D, 0x38B293F3, + 0x7413C95F, 0xD1525B21, 0x3B7C9B52, 0x9E3D092C, + 0xEACD6D45, 0x4F8CFF3B, 0xA5A23F48, 0x00E3AD36, + 0x3CE08A10, 0x99A1186E, 0x738FD81D, 0xD6CE4A63, + 0xA23E2E0A, 0x077FBC74, 0xED517C07, 0x4810EE79, + 0x04B1B4D5, 0xA1F026AB, 0x4BDEE6D8, 0xEE9F74A6, + 0x9A6F10CF, 0x3F2E82B1, 0xD50042C2, 0x7041D0BC, + 0xAD060C8E, 0x08479EF0, 0xE2695E83, 0x4728CCFD, + 0x33D8A894, 0x96993AEA, 0x7CB7FA99, 0xD9F668E7, + 0x9557324B, 0x3016A035, 0xDA386046, 0x7F79F238, + 0x0B899651, 0xAEC8042F, 0x44E6C45C, 0xE1A75622, + 0xDDA47104, 0x78E5E37A, 0x92CB2309, 0x378AB177, + 0x437AD51E, 0xE63B4760, 0x0C158713, 0xA954156D, + 0xE5F54FC1, 0x40B4DDBF, 0xAA9A1DCC, 0x0FDB8FB2, + 0x7B2BEBDB, 0xDE6A79A5, 0x3444B9D6, 0x91052BA8 +}; +const uint32_t CRC32C_T8_3[256] = { + 0x00000000, 0xDD45AAB8, 0xBF672381, 0x62228939, + 0x7B2231F3, 0xA6679B4B, 0xC4451272, 0x1900B8CA, + 0xF64463E6, 0x2B01C95E, 0x49234067, 0x9466EADF, + 0x8D665215, 0x5023F8AD, 0x32017194, 0xEF44DB2C, + 0xE964B13D, 0x34211B85, 0x560392BC, 0x8B463804, + 0x924680CE, 0x4F032A76, 0x2D21A34F, 0xF06409F7, + 0x1F20D2DB, 0xC2657863, 0xA047F15A, 0x7D025BE2, + 0x6402E328, 0xB9474990, 0xDB65C0A9, 0x06206A11, + 0xD725148B, 0x0A60BE33, 0x6842370A, 0xB5079DB2, + 0xAC072578, 0x71428FC0, 0x136006F9, 0xCE25AC41, + 0x2161776D, 0xFC24DDD5, 0x9E0654EC, 0x4343FE54, + 0x5A43469E, 0x8706EC26, 0xE524651F, 0x3861CFA7, + 0x3E41A5B6, 0xE3040F0E, 0x81268637, 0x5C632C8F, + 0x45639445, 0x98263EFD, 0xFA04B7C4, 0x27411D7C, + 0xC805C650, 0x15406CE8, 0x7762E5D1, 0xAA274F69, + 0xB327F7A3, 0x6E625D1B, 0x0C40D422, 0xD1057E9A, + 0xABA65FE7, 0x76E3F55F, 0x14C17C66, 0xC984D6DE, + 0xD0846E14, 0x0DC1C4AC, 0x6FE34D95, 0xB2A6E72D, + 0x5DE23C01, 0x80A796B9, 0xE2851F80, 0x3FC0B538, + 0x26C00DF2, 0xFB85A74A, 0x99A72E73, 0x44E284CB, + 0x42C2EEDA, 0x9F874462, 0xFDA5CD5B, 0x20E067E3, + 0x39E0DF29, 0xE4A57591, 0x8687FCA8, 0x5BC25610, + 0xB4868D3C, 0x69C32784, 0x0BE1AEBD, 0xD6A40405, + 0xCFA4BCCF, 0x12E11677, 0x70C39F4E, 0xAD8635F6, + 0x7C834B6C, 0xA1C6E1D4, 0xC3E468ED, 0x1EA1C255, + 0x07A17A9F, 0xDAE4D027, 0xB8C6591E, 0x6583F3A6, + 0x8AC7288A, 0x57828232, 0x35A00B0B, 0xE8E5A1B3, + 0xF1E51979, 0x2CA0B3C1, 0x4E823AF8, 0x93C79040, + 0x95E7FA51, 0x48A250E9, 0x2A80D9D0, 0xF7C57368, + 0xEEC5CBA2, 0x3380611A, 0x51A2E823, 0x8CE7429B, + 0x63A399B7, 0xBEE6330F, 0xDCC4BA36, 0x0181108E, + 0x1881A844, 0xC5C402FC, 0xA7E68BC5, 0x7AA3217D, + 0x52A0C93F, 0x8FE56387, 0xEDC7EABE, 0x30824006, + 0x2982F8CC, 0xF4C75274, 0x96E5DB4D, 0x4BA071F5, + 0xA4E4AAD9, 0x79A10061, 0x1B838958, 0xC6C623E0, + 0xDFC69B2A, 0x02833192, 0x60A1B8AB, 0xBDE41213, + 0xBBC47802, 0x6681D2BA, 0x04A35B83, 0xD9E6F13B, + 0xC0E649F1, 0x1DA3E349, 0x7F816A70, 0xA2C4C0C8, + 0x4D801BE4, 0x90C5B15C, 0xF2E73865, 0x2FA292DD, + 0x36A22A17, 0xEBE780AF, 0x89C50996, 0x5480A32E, + 0x8585DDB4, 0x58C0770C, 0x3AE2FE35, 0xE7A7548D, + 0xFEA7EC47, 0x23E246FF, 0x41C0CFC6, 0x9C85657E, + 0x73C1BE52, 0xAE8414EA, 0xCCA69DD3, 0x11E3376B, + 0x08E38FA1, 0xD5A62519, 0xB784AC20, 0x6AC10698, + 0x6CE16C89, 0xB1A4C631, 0xD3864F08, 0x0EC3E5B0, + 0x17C35D7A, 0xCA86F7C2, 0xA8A47EFB, 0x75E1D443, + 0x9AA50F6F, 0x47E0A5D7, 0x25C22CEE, 0xF8878656, + 0xE1873E9C, 0x3CC29424, 0x5EE01D1D, 0x83A5B7A5, + 0xF90696D8, 0x24433C60, 0x4661B559, 0x9B241FE1, + 0x8224A72B, 0x5F610D93, 0x3D4384AA, 0xE0062E12, + 0x0F42F53E, 0xD2075F86, 0xB025D6BF, 0x6D607C07, + 0x7460C4CD, 0xA9256E75, 0xCB07E74C, 0x16424DF4, + 0x106227E5, 0xCD278D5D, 0xAF050464, 0x7240AEDC, + 0x6B401616, 0xB605BCAE, 0xD4273597, 0x09629F2F, + 0xE6264403, 0x3B63EEBB, 0x59416782, 0x8404CD3A, + 0x9D0475F0, 0x4041DF48, 0x22635671, 0xFF26FCC9, + 0x2E238253, 0xF36628EB, 0x9144A1D2, 0x4C010B6A, + 0x5501B3A0, 0x88441918, 0xEA669021, 0x37233A99, + 0xD867E1B5, 0x05224B0D, 0x6700C234, 0xBA45688C, + 0xA345D046, 0x7E007AFE, 0x1C22F3C7, 0xC167597F, + 0xC747336E, 0x1A0299D6, 0x782010EF, 0xA565BA57, + 0xBC65029D, 0x6120A825, 0x0302211C, 0xDE478BA4, + 0x31035088, 0xEC46FA30, 0x8E647309, 0x5321D9B1, + 0x4A21617B, 0x9764CBC3, 0xF54642FA, 0x2803E842 +}; +const uint32_t CRC32C_T8_4[256] = { + 0x00000000, 0x38116FAC, 0x7022DF58, 0x4833B0F4, + 0xE045BEB0, 0xD854D11C, 0x906761E8, 0xA8760E44, + 0xC5670B91, 0xFD76643D, 0xB545D4C9, 0x8D54BB65, + 0x2522B521, 0x1D33DA8D, 0x55006A79, 0x6D1105D5, + 0x8F2261D3, 0xB7330E7F, 0xFF00BE8B, 0xC711D127, + 0x6F67DF63, 0x5776B0CF, 0x1F45003B, 0x27546F97, + 0x4A456A42, 0x725405EE, 0x3A67B51A, 0x0276DAB6, + 0xAA00D4F2, 0x9211BB5E, 0xDA220BAA, 0xE2336406, + 0x1BA8B557, 0x23B9DAFB, 0x6B8A6A0F, 0x539B05A3, + 0xFBED0BE7, 0xC3FC644B, 0x8BCFD4BF, 0xB3DEBB13, + 0xDECFBEC6, 0xE6DED16A, 0xAEED619E, 0x96FC0E32, + 0x3E8A0076, 0x069B6FDA, 0x4EA8DF2E, 0x76B9B082, + 0x948AD484, 0xAC9BBB28, 0xE4A80BDC, 0xDCB96470, + 0x74CF6A34, 0x4CDE0598, 0x04EDB56C, 0x3CFCDAC0, + 0x51EDDF15, 0x69FCB0B9, 0x21CF004D, 0x19DE6FE1, + 0xB1A861A5, 0x89B90E09, 0xC18ABEFD, 0xF99BD151, + 0x37516AAE, 0x0F400502, 0x4773B5F6, 0x7F62DA5A, + 0xD714D41E, 0xEF05BBB2, 0xA7360B46, 0x9F2764EA, + 0xF236613F, 0xCA270E93, 0x8214BE67, 0xBA05D1CB, + 0x1273DF8F, 0x2A62B023, 0x625100D7, 0x5A406F7B, + 0xB8730B7D, 0x806264D1, 0xC851D425, 0xF040BB89, + 0x5836B5CD, 0x6027DA61, 0x28146A95, 0x10050539, + 0x7D1400EC, 0x45056F40, 0x0D36DFB4, 0x3527B018, + 0x9D51BE5C, 0xA540D1F0, 0xED736104, 0xD5620EA8, + 0x2CF9DFF9, 0x14E8B055, 0x5CDB00A1, 0x64CA6F0D, + 0xCCBC6149, 0xF4AD0EE5, 0xBC9EBE11, 0x848FD1BD, + 0xE99ED468, 0xD18FBBC4, 0x99BC0B30, 0xA1AD649C, + 0x09DB6AD8, 0x31CA0574, 0x79F9B580, 0x41E8DA2C, + 0xA3DBBE2A, 0x9BCAD186, 0xD3F96172, 0xEBE80EDE, + 0x439E009A, 0x7B8F6F36, 0x33BCDFC2, 0x0BADB06E, + 0x66BCB5BB, 0x5EADDA17, 0x169E6AE3, 0x2E8F054F, + 0x86F90B0B, 0xBEE864A7, 0xF6DBD453, 0xCECABBFF, + 0x6EA2D55C, 0x56B3BAF0, 0x1E800A04, 0x269165A8, + 0x8EE76BEC, 0xB6F60440, 0xFEC5B4B4, 0xC6D4DB18, + 0xABC5DECD, 0x93D4B161, 0xDBE70195, 0xE3F66E39, + 0x4B80607D, 0x73910FD1, 0x3BA2BF25, 0x03B3D089, + 0xE180B48F, 0xD991DB23, 0x91A26BD7, 0xA9B3047B, + 0x01C50A3F, 0x39D46593, 0x71E7D567, 0x49F6BACB, + 0x24E7BF1E, 0x1CF6D0B2, 0x54C56046, 0x6CD40FEA, + 0xC4A201AE, 0xFCB36E02, 0xB480DEF6, 0x8C91B15A, + 0x750A600B, 0x4D1B0FA7, 0x0528BF53, 0x3D39D0FF, + 0x954FDEBB, 0xAD5EB117, 0xE56D01E3, 0xDD7C6E4F, + 0xB06D6B9A, 0x887C0436, 0xC04FB4C2, 0xF85EDB6E, + 0x5028D52A, 0x6839BA86, 0x200A0A72, 0x181B65DE, + 0xFA2801D8, 0xC2396E74, 0x8A0ADE80, 0xB21BB12C, + 0x1A6DBF68, 0x227CD0C4, 0x6A4F6030, 0x525E0F9C, + 0x3F4F0A49, 0x075E65E5, 0x4F6DD511, 0x777CBABD, + 0xDF0AB4F9, 0xE71BDB55, 0xAF286BA1, 0x9739040D, + 0x59F3BFF2, 0x61E2D05E, 0x29D160AA, 0x11C00F06, + 0xB9B60142, 0x81A76EEE, 0xC994DE1A, 0xF185B1B6, + 0x9C94B463, 0xA485DBCF, 0xECB66B3B, 0xD4A70497, + 0x7CD10AD3, 0x44C0657F, 0x0CF3D58B, 0x34E2BA27, + 0xD6D1DE21, 0xEEC0B18D, 0xA6F30179, 0x9EE26ED5, + 0x36946091, 0x0E850F3D, 0x46B6BFC9, 0x7EA7D065, + 0x13B6D5B0, 0x2BA7BA1C, 0x63940AE8, 0x5B856544, + 0xF3F36B00, 0xCBE204AC, 0x83D1B458, 0xBBC0DBF4, + 0x425B0AA5, 0x7A4A6509, 0x3279D5FD, 0x0A68BA51, + 0xA21EB415, 0x9A0FDBB9, 0xD23C6B4D, 0xEA2D04E1, + 0x873C0134, 0xBF2D6E98, 0xF71EDE6C, 0xCF0FB1C0, + 0x6779BF84, 0x5F68D028, 0x175B60DC, 0x2F4A0F70, + 0xCD796B76, 0xF56804DA, 0xBD5BB42E, 0x854ADB82, + 0x2D3CD5C6, 0x152DBA6A, 0x5D1E0A9E, 0x650F6532, + 0x081E60E7, 0x300F0F4B, 0x783CBFBF, 0x402DD013, + 0xE85BDE57, 0xD04AB1FB, 0x9879010F, 0xA0686EA3 +}; +const uint32_t CRC32C_T8_5[256] = { + 0x00000000, 0xEF306B19, 0xDB8CA0C3, 0x34BCCBDA, + 0xB2F53777, 0x5DC55C6E, 0x697997B4, 0x8649FCAD, + 0x6006181F, 0x8F367306, 0xBB8AB8DC, 0x54BAD3C5, + 0xD2F32F68, 0x3DC34471, 0x097F8FAB, 0xE64FE4B2, + 0xC00C303E, 0x2F3C5B27, 0x1B8090FD, 0xF4B0FBE4, + 0x72F90749, 0x9DC96C50, 0xA975A78A, 0x4645CC93, + 0xA00A2821, 0x4F3A4338, 0x7B8688E2, 0x94B6E3FB, + 0x12FF1F56, 0xFDCF744F, 0xC973BF95, 0x2643D48C, + 0x85F4168D, 0x6AC47D94, 0x5E78B64E, 0xB148DD57, + 0x370121FA, 0xD8314AE3, 0xEC8D8139, 0x03BDEA20, + 0xE5F20E92, 0x0AC2658B, 0x3E7EAE51, 0xD14EC548, + 0x570739E5, 0xB83752FC, 0x8C8B9926, 0x63BBF23F, + 0x45F826B3, 0xAAC84DAA, 0x9E748670, 0x7144ED69, + 0xF70D11C4, 0x183D7ADD, 0x2C81B107, 0xC3B1DA1E, + 0x25FE3EAC, 0xCACE55B5, 0xFE729E6F, 0x1142F576, + 0x970B09DB, 0x783B62C2, 0x4C87A918, 0xA3B7C201, + 0x0E045BEB, 0xE13430F2, 0xD588FB28, 0x3AB89031, + 0xBCF16C9C, 0x53C10785, 0x677DCC5F, 0x884DA746, + 0x6E0243F4, 0x813228ED, 0xB58EE337, 0x5ABE882E, + 0xDCF77483, 0x33C71F9A, 0x077BD440, 0xE84BBF59, + 0xCE086BD5, 0x213800CC, 0x1584CB16, 0xFAB4A00F, + 0x7CFD5CA2, 0x93CD37BB, 0xA771FC61, 0x48419778, + 0xAE0E73CA, 0x413E18D3, 0x7582D309, 0x9AB2B810, + 0x1CFB44BD, 0xF3CB2FA4, 0xC777E47E, 0x28478F67, + 0x8BF04D66, 0x64C0267F, 0x507CEDA5, 0xBF4C86BC, + 0x39057A11, 0xD6351108, 0xE289DAD2, 0x0DB9B1CB, + 0xEBF65579, 0x04C63E60, 0x307AF5BA, 0xDF4A9EA3, + 0x5903620E, 0xB6330917, 0x828FC2CD, 0x6DBFA9D4, + 0x4BFC7D58, 0xA4CC1641, 0x9070DD9B, 0x7F40B682, + 0xF9094A2F, 0x16392136, 0x2285EAEC, 0xCDB581F5, + 0x2BFA6547, 0xC4CA0E5E, 0xF076C584, 0x1F46AE9D, + 0x990F5230, 0x763F3929, 0x4283F2F3, 0xADB399EA, + 0x1C08B7D6, 0xF338DCCF, 0xC7841715, 0x28B47C0C, + 0xAEFD80A1, 0x41CDEBB8, 0x75712062, 0x9A414B7B, + 0x7C0EAFC9, 0x933EC4D0, 0xA7820F0A, 0x48B26413, + 0xCEFB98BE, 0x21CBF3A7, 0x1577387D, 0xFA475364, + 0xDC0487E8, 0x3334ECF1, 0x0788272B, 0xE8B84C32, + 0x6EF1B09F, 0x81C1DB86, 0xB57D105C, 0x5A4D7B45, + 0xBC029FF7, 0x5332F4EE, 0x678E3F34, 0x88BE542D, + 0x0EF7A880, 0xE1C7C399, 0xD57B0843, 0x3A4B635A, + 0x99FCA15B, 0x76CCCA42, 0x42700198, 0xAD406A81, + 0x2B09962C, 0xC439FD35, 0xF08536EF, 0x1FB55DF6, + 0xF9FAB944, 0x16CAD25D, 0x22761987, 0xCD46729E, + 0x4B0F8E33, 0xA43FE52A, 0x90832EF0, 0x7FB345E9, + 0x59F09165, 0xB6C0FA7C, 0x827C31A6, 0x6D4C5ABF, + 0xEB05A612, 0x0435CD0B, 0x308906D1, 0xDFB96DC8, + 0x39F6897A, 0xD6C6E263, 0xE27A29B9, 0x0D4A42A0, + 0x8B03BE0D, 0x6433D514, 0x508F1ECE, 0xBFBF75D7, + 0x120CEC3D, 0xFD3C8724, 0xC9804CFE, 0x26B027E7, + 0xA0F9DB4A, 0x4FC9B053, 0x7B757B89, 0x94451090, + 0x720AF422, 0x9D3A9F3B, 0xA98654E1, 0x46B63FF8, + 0xC0FFC355, 0x2FCFA84C, 0x1B736396, 0xF443088F, + 0xD200DC03, 0x3D30B71A, 0x098C7CC0, 0xE6BC17D9, + 0x60F5EB74, 0x8FC5806D, 0xBB794BB7, 0x544920AE, + 0xB206C41C, 0x5D36AF05, 0x698A64DF, 0x86BA0FC6, + 0x00F3F36B, 0xEFC39872, 0xDB7F53A8, 0x344F38B1, + 0x97F8FAB0, 0x78C891A9, 0x4C745A73, 0xA344316A, + 0x250DCDC7, 0xCA3DA6DE, 0xFE816D04, 0x11B1061D, + 0xF7FEE2AF, 0x18CE89B6, 0x2C72426C, 0xC3422975, + 0x450BD5D8, 0xAA3BBEC1, 0x9E87751B, 0x71B71E02, + 0x57F4CA8E, 0xB8C4A197, 0x8C786A4D, 0x63480154, + 0xE501FDF9, 0x0A3196E0, 0x3E8D5D3A, 0xD1BD3623, + 0x37F2D291, 0xD8C2B988, 0xEC7E7252, 0x034E194B, + 0x8507E5E6, 0x6A378EFF, 0x5E8B4525, 0xB1BB2E3C +}; +const uint32_t CRC32C_T8_6[256] = { + 0x00000000, 0x68032CC8, 0xD0065990, 0xB8057558, + 0xA5E0C5D1, 0xCDE3E919, 0x75E69C41, 0x1DE5B089, + 0x4E2DFD53, 0x262ED19B, 0x9E2BA4C3, 0xF628880B, + 0xEBCD3882, 0x83CE144A, 0x3BCB6112, 0x53C84DDA, + 0x9C5BFAA6, 0xF458D66E, 0x4C5DA336, 0x245E8FFE, + 0x39BB3F77, 0x51B813BF, 0xE9BD66E7, 0x81BE4A2F, + 0xD27607F5, 0xBA752B3D, 0x02705E65, 0x6A7372AD, + 0x7796C224, 0x1F95EEEC, 0xA7909BB4, 0xCF93B77C, + 0x3D5B83BD, 0x5558AF75, 0xED5DDA2D, 0x855EF6E5, + 0x98BB466C, 0xF0B86AA4, 0x48BD1FFC, 0x20BE3334, + 0x73767EEE, 0x1B755226, 0xA370277E, 0xCB730BB6, + 0xD696BB3F, 0xBE9597F7, 0x0690E2AF, 0x6E93CE67, + 0xA100791B, 0xC90355D3, 0x7106208B, 0x19050C43, + 0x04E0BCCA, 0x6CE39002, 0xD4E6E55A, 0xBCE5C992, + 0xEF2D8448, 0x872EA880, 0x3F2BDDD8, 0x5728F110, + 0x4ACD4199, 0x22CE6D51, 0x9ACB1809, 0xF2C834C1, + 0x7AB7077A, 0x12B42BB2, 0xAAB15EEA, 0xC2B27222, + 0xDF57C2AB, 0xB754EE63, 0x0F519B3B, 0x6752B7F3, + 0x349AFA29, 0x5C99D6E1, 0xE49CA3B9, 0x8C9F8F71, + 0x917A3FF8, 0xF9791330, 0x417C6668, 0x297F4AA0, + 0xE6ECFDDC, 0x8EEFD114, 0x36EAA44C, 0x5EE98884, + 0x430C380D, 0x2B0F14C5, 0x930A619D, 0xFB094D55, + 0xA8C1008F, 0xC0C22C47, 0x78C7591F, 0x10C475D7, + 0x0D21C55E, 0x6522E996, 0xDD279CCE, 0xB524B006, + 0x47EC84C7, 0x2FEFA80F, 0x97EADD57, 0xFFE9F19F, + 0xE20C4116, 0x8A0F6DDE, 0x320A1886, 0x5A09344E, + 0x09C17994, 0x61C2555C, 0xD9C72004, 0xB1C40CCC, + 0xAC21BC45, 0xC422908D, 0x7C27E5D5, 0x1424C91D, + 0xDBB77E61, 0xB3B452A9, 0x0BB127F1, 0x63B20B39, + 0x7E57BBB0, 0x16549778, 0xAE51E220, 0xC652CEE8, + 0x959A8332, 0xFD99AFFA, 0x459CDAA2, 0x2D9FF66A, + 0x307A46E3, 0x58796A2B, 0xE07C1F73, 0x887F33BB, + 0xF56E0EF4, 0x9D6D223C, 0x25685764, 0x4D6B7BAC, + 0x508ECB25, 0x388DE7ED, 0x808892B5, 0xE88BBE7D, + 0xBB43F3A7, 0xD340DF6F, 0x6B45AA37, 0x034686FF, + 0x1EA33676, 0x76A01ABE, 0xCEA56FE6, 0xA6A6432E, + 0x6935F452, 0x0136D89A, 0xB933ADC2, 0xD130810A, + 0xCCD53183, 0xA4D61D4B, 0x1CD36813, 0x74D044DB, + 0x27180901, 0x4F1B25C9, 0xF71E5091, 0x9F1D7C59, + 0x82F8CCD0, 0xEAFBE018, 0x52FE9540, 0x3AFDB988, + 0xC8358D49, 0xA036A181, 0x1833D4D9, 0x7030F811, + 0x6DD54898, 0x05D66450, 0xBDD31108, 0xD5D03DC0, + 0x8618701A, 0xEE1B5CD2, 0x561E298A, 0x3E1D0542, + 0x23F8B5CB, 0x4BFB9903, 0xF3FEEC5B, 0x9BFDC093, + 0x546E77EF, 0x3C6D5B27, 0x84682E7F, 0xEC6B02B7, + 0xF18EB23E, 0x998D9EF6, 0x2188EBAE, 0x498BC766, + 0x1A438ABC, 0x7240A674, 0xCA45D32C, 0xA246FFE4, + 0xBFA34F6D, 0xD7A063A5, 0x6FA516FD, 0x07A63A35, + 0x8FD9098E, 0xE7DA2546, 0x5FDF501E, 0x37DC7CD6, + 0x2A39CC5F, 0x423AE097, 0xFA3F95CF, 0x923CB907, + 0xC1F4F4DD, 0xA9F7D815, 0x11F2AD4D, 0x79F18185, + 0x6414310C, 0x0C171DC4, 0xB412689C, 0xDC114454, + 0x1382F328, 0x7B81DFE0, 0xC384AAB8, 0xAB878670, + 0xB66236F9, 0xDE611A31, 0x66646F69, 0x0E6743A1, + 0x5DAF0E7B, 0x35AC22B3, 0x8DA957EB, 0xE5AA7B23, + 0xF84FCBAA, 0x904CE762, 0x2849923A, 0x404ABEF2, + 0xB2828A33, 0xDA81A6FB, 0x6284D3A3, 0x0A87FF6B, + 0x17624FE2, 0x7F61632A, 0xC7641672, 0xAF673ABA, + 0xFCAF7760, 0x94AC5BA8, 0x2CA92EF0, 0x44AA0238, + 0x594FB2B1, 0x314C9E79, 0x8949EB21, 0xE14AC7E9, + 0x2ED97095, 0x46DA5C5D, 0xFEDF2905, 0x96DC05CD, + 0x8B39B544, 0xE33A998C, 0x5B3FECD4, 0x333CC01C, + 0x60F48DC6, 0x08F7A10E, 0xB0F2D456, 0xD8F1F89E, + 0xC5144817, 0xAD1764DF, 0x15121187, 0x7D113D4F +}; +const uint32_t CRC32C_T8_7[256] = { + 0x00000000, 0x493C7D27, 0x9278FA4E, 0xDB448769, + 0x211D826D, 0x6821FF4A, 0xB3657823, 0xFA590504, + 0x423B04DA, 0x0B0779FD, 0xD043FE94, 0x997F83B3, + 0x632686B7, 0x2A1AFB90, 0xF15E7CF9, 0xB86201DE, + 0x847609B4, 0xCD4A7493, 0x160EF3FA, 0x5F328EDD, + 0xA56B8BD9, 0xEC57F6FE, 0x37137197, 0x7E2F0CB0, + 0xC64D0D6E, 0x8F717049, 0x5435F720, 0x1D098A07, + 0xE7508F03, 0xAE6CF224, 0x7528754D, 0x3C14086A, + 0x0D006599, 0x443C18BE, 0x9F789FD7, 0xD644E2F0, + 0x2C1DE7F4, 0x65219AD3, 0xBE651DBA, 0xF759609D, + 0x4F3B6143, 0x06071C64, 0xDD439B0D, 0x947FE62A, + 0x6E26E32E, 0x271A9E09, 0xFC5E1960, 0xB5626447, + 0x89766C2D, 0xC04A110A, 0x1B0E9663, 0x5232EB44, + 0xA86BEE40, 0xE1579367, 0x3A13140E, 0x732F6929, + 0xCB4D68F7, 0x827115D0, 0x593592B9, 0x1009EF9E, + 0xEA50EA9A, 0xA36C97BD, 0x782810D4, 0x31146DF3, + 0x1A00CB32, 0x533CB615, 0x8878317C, 0xC1444C5B, + 0x3B1D495F, 0x72213478, 0xA965B311, 0xE059CE36, + 0x583BCFE8, 0x1107B2CF, 0xCA4335A6, 0x837F4881, + 0x79264D85, 0x301A30A2, 0xEB5EB7CB, 0xA262CAEC, + 0x9E76C286, 0xD74ABFA1, 0x0C0E38C8, 0x453245EF, + 0xBF6B40EB, 0xF6573DCC, 0x2D13BAA5, 0x642FC782, + 0xDC4DC65C, 0x9571BB7B, 0x4E353C12, 0x07094135, + 0xFD504431, 0xB46C3916, 0x6F28BE7F, 0x2614C358, + 0x1700AEAB, 0x5E3CD38C, 0x857854E5, 0xCC4429C2, + 0x361D2CC6, 0x7F2151E1, 0xA465D688, 0xED59ABAF, + 0x553BAA71, 0x1C07D756, 0xC743503F, 0x8E7F2D18, + 0x7426281C, 0x3D1A553B, 0xE65ED252, 0xAF62AF75, + 0x9376A71F, 0xDA4ADA38, 0x010E5D51, 0x48322076, + 0xB26B2572, 0xFB575855, 0x2013DF3C, 0x692FA21B, + 0xD14DA3C5, 0x9871DEE2, 0x4335598B, 0x0A0924AC, + 0xF05021A8, 0xB96C5C8F, 0x6228DBE6, 0x2B14A6C1, + 0x34019664, 0x7D3DEB43, 0xA6796C2A, 0xEF45110D, + 0x151C1409, 0x5C20692E, 0x8764EE47, 0xCE589360, + 0x763A92BE, 0x3F06EF99, 0xE44268F0, 0xAD7E15D7, + 0x572710D3, 0x1E1B6DF4, 0xC55FEA9D, 0x8C6397BA, + 0xB0779FD0, 0xF94BE2F7, 0x220F659E, 0x6B3318B9, + 0x916A1DBD, 0xD856609A, 0x0312E7F3, 0x4A2E9AD4, + 0xF24C9B0A, 0xBB70E62D, 0x60346144, 0x29081C63, + 0xD3511967, 0x9A6D6440, 0x4129E329, 0x08159E0E, + 0x3901F3FD, 0x703D8EDA, 0xAB7909B3, 0xE2457494, + 0x181C7190, 0x51200CB7, 0x8A648BDE, 0xC358F6F9, + 0x7B3AF727, 0x32068A00, 0xE9420D69, 0xA07E704E, + 0x5A27754A, 0x131B086D, 0xC85F8F04, 0x8163F223, + 0xBD77FA49, 0xF44B876E, 0x2F0F0007, 0x66337D20, + 0x9C6A7824, 0xD5560503, 0x0E12826A, 0x472EFF4D, + 0xFF4CFE93, 0xB67083B4, 0x6D3404DD, 0x240879FA, + 0xDE517CFE, 0x976D01D9, 0x4C2986B0, 0x0515FB97, + 0x2E015D56, 0x673D2071, 0xBC79A718, 0xF545DA3F, + 0x0F1CDF3B, 0x4620A21C, 0x9D642575, 0xD4585852, + 0x6C3A598C, 0x250624AB, 0xFE42A3C2, 0xB77EDEE5, + 0x4D27DBE1, 0x041BA6C6, 0xDF5F21AF, 0x96635C88, + 0xAA7754E2, 0xE34B29C5, 0x380FAEAC, 0x7133D38B, + 0x8B6AD68F, 0xC256ABA8, 0x19122CC1, 0x502E51E6, + 0xE84C5038, 0xA1702D1F, 0x7A34AA76, 0x3308D751, + 0xC951D255, 0x806DAF72, 0x5B29281B, 0x1215553C, + 0x230138CF, 0x6A3D45E8, 0xB179C281, 0xF845BFA6, + 0x021CBAA2, 0x4B20C785, 0x906440EC, 0xD9583DCB, + 0x613A3C15, 0x28064132, 0xF342C65B, 0xBA7EBB7C, + 0x4027BE78, 0x091BC35F, 0xD25F4436, 0x9B633911, + 0xA777317B, 0xEE4B4C5C, 0x350FCB35, 0x7C33B612, + 0x866AB316, 0xCF56CE31, 0x14124958, 0x5D2E347F, + 0xE54C35A1, 0xAC704886, 0x7734CFEF, 0x3E08B2C8, + 0xC451B7CC, 0x8D6DCAEB, 0x56294D82, 0x1F1530A5 +}; diff --git a/hadoop-common/src/main/native/src/org/apache/hadoop/util/gcc_optimizations.h b/hadoop-common/src/main/native/src/org/apache/hadoop/util/gcc_optimizations.h new file mode 100644 index 00000000000..8c0fbffe6a1 --- /dev/null +++ b/hadoop-common/src/main/native/src/org/apache/hadoop/util/gcc_optimizations.h @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __GCC_OPTIMIZATIONS_H_INCLUDED +#define __GCC_OPTIMIZATIONS_H_INCLUDED + +// Hints to gcc optimizer -- compiled out on non-GCC +#ifdef __GNUC__ +#define likely(x) __builtin_expect((x),1) +#define unlikely(x) __builtin_expect((x),0) +#else +#define likely(x) (x) +#define unlikely(x) (x) +#endif + +#endif diff --git a/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java b/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java index b7b764e1424..ef2e18858c6 100644 --- a/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java +++ b/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java @@ -126,6 +126,8 @@ public class TestFilterFileSystem extends TestCase { public void moveFromLocalFile(Path[] srcs, Path dst) { } public void moveFromLocalFile(Path src, Path dst) { } public void copyToLocalFile(Path src, Path dst) { } + public void copyToLocalFile(boolean delSrc, Path src, Path dst, + boolean useRawLocalFileSystem) { } public void moveToLocalFile(Path src, Path dst) { } public long getBlockSize(Path f) { return 0; } public FSDataOutputStream primitiveCreate(final Path f, diff --git a/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java b/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java index 703e7691848..d36a39edb8f 100644 --- a/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java +++ b/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java @@ -185,6 +185,48 @@ public class TestPath extends TestCase { assertEquals("foo://bar/fud#boo", new Path(new Path(new URI( "foo://bar/baz#bud")), new Path(new URI("/fud#boo"))).toString()); } + + /** Test URIs created from Path objects */ + public void testPathToUriConversion() throws URISyntaxException, IOException { + // Path differs from URI in that it ignores the query part.. + assertEquals(new URI(null, null, "/foo?bar", null, null), new Path("/foo?bar").toUri()); + assertEquals(new URI(null, null, "/foo\"bar", null, null), new Path("/foo\"bar").toUri()); + assertEquals(new URI(null, null, "/foo bar", null, null), new Path("/foo bar").toUri()); + // therefore "foo?bar" is a valid Path, so a URI created from a Path has path "foo?bar" + // where in a straight URI the path part is just "foo" + assertEquals("/foo?bar", new Path("http://localhost/foo?bar").toUri().getPath()); + assertEquals("/foo", new URI("http://localhost/foo?bar").getPath()); + + // The path part handling in Path is equivalent to URI + assertEquals(new URI("/foo;bar").getPath(), new Path("/foo;bar").toUri().getPath()); + assertEquals(new URI("/foo;bar"), new Path("/foo;bar").toUri()); + assertEquals(new URI("/foo+bar"), new Path("/foo+bar").toUri()); + assertEquals(new URI("/foo-bar"), new Path("/foo-bar").toUri()); + assertEquals(new URI("/foo=bar"), new Path("/foo=bar").toUri()); + assertEquals(new URI("/foo,bar"), new Path("/foo,bar").toUri()); + } + + /** Test reserved characters in URIs (and therefore Paths) */ + public void testReservedCharacters() throws URISyntaxException, IOException { + // URI encodes the path + assertEquals("/foo%20bar", new URI(null, null, "/foo bar", null, null).getRawPath()); + // URI#getPath decodes the path + assertEquals("/foo bar", new URI(null, null, "/foo bar", null, null).getPath()); + // URI#toString returns an encoded path + assertEquals("/foo%20bar", new URI(null, null, "/foo bar", null, null).toString()); + assertEquals("/foo%20bar", new Path("/foo bar").toUri().toString()); + // Reserved chars are not encoded + assertEquals("/foo;bar", new URI("/foo;bar").getPath()); + assertEquals("/foo;bar", new URI("/foo;bar").getRawPath()); + assertEquals("/foo+bar", new URI("/foo+bar").getPath()); + assertEquals("/foo+bar", new URI("/foo+bar").getRawPath()); + + // URI#getPath decodes the path part (and URL#getPath does not decode) + assertEquals("/foo bar", new Path("http://localhost/foo bar").toUri().getPath()); + assertEquals("/foo%20bar", new Path("http://localhost/foo bar").toUri().toURL().getPath()); + assertEquals("/foo?bar", new URI("http", "localhost", "/foo?bar", null, null).getPath()); + assertEquals("/foo%3Fbar", new URI("http", "localhost", "/foo?bar", null, null).toURL().getPath()); + } public void testMakeQualified() throws URISyntaxException { URI defaultUri = new URI("hdfs://host1/dir1"); diff --git a/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java b/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java index 50bc1e10a53..d1783f28aea 100644 --- a/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java +++ b/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java @@ -501,7 +501,7 @@ public class TestCodec { LOG.info("SUCCESS! Completed SequenceFileCodecTest with codec \"" + codecClass + "\""); } - public static void main(String[] args) { + public static void main(String[] args) throws IOException { int count = 10000; String codecClass = "org.apache.hadoop.io.compress.DefaultCodec"; @@ -511,25 +511,20 @@ public class TestCodec { System.exit(-1); } - try { - for (int i=0; i < args.length; ++i) { // parse command line - if (args[i] == null) { - continue; - } else if (args[i].equals("-count")) { - count = Integer.parseInt(args[++i]); - } else if (args[i].equals("-codec")) { - codecClass = args[++i]; - } + for (int i=0; i < args.length; ++i) { // parse command line + if (args[i] == null) { + continue; + } else if (args[i].equals("-count")) { + count = Integer.parseInt(args[++i]); + } else if (args[i].equals("-codec")) { + codecClass = args[++i]; } - - Configuration conf = new Configuration(); - int seed = 0; - codecTest(conf, seed, count, codecClass); - } catch (Exception e) { - System.err.println("Caught: " + e); - e.printStackTrace(); } + Configuration conf = new Configuration(); + int seed = 0; + // Note that exceptions will propagate out. + codecTest(conf, seed, count, codecClass); } @Test diff --git a/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java b/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java index 0974aa335d6..c3923ed9510 100644 --- a/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java +++ b/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java @@ -18,7 +18,6 @@ package org.apache.hadoop.net; import java.util.*; -import java.net.UnknownHostException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; @@ -60,19 +59,4 @@ public class StaticMapping extends Configured implements DNSToSwitchMapping { return m; } } - public List resolveValidHosts(List names) - throws UnknownHostException { - List m = new ArrayList(); - synchronized (nameToRackMap) { - for (String name : names) { - String rackId; - if ((rackId = nameToRackMap.get(name)) != null) { - m.add(rackId); - } else { - throw new UnknownHostException(name); - } - } - return m; - } - } } diff --git a/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java b/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java index 2aad71cb52e..f49d4c886ec 100644 --- a/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java +++ b/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java @@ -72,4 +72,20 @@ public class TestNetUtils { assertNull(NetUtils.getLocalInetAddress("invalid-address-for-test")); assertNull(NetUtils.getLocalInetAddress(null)); } + + @Test(expected=UnknownHostException.class) + public void testVerifyHostnamesException() throws UnknownHostException { + String[] names = {"valid.host.com", "1.com", "invalid host here"}; + NetUtils.verifyHostnames(names); + } + + @Test + public void testVerifyHostnamesNoException() { + String[] names = {"valid.host.com", "1.com"}; + try { + NetUtils.verifyHostnames(names); + } catch (UnknownHostException e) { + fail("NetUtils.verifyHostnames threw unexpected UnknownHostException"); + } + } } diff --git a/hadoop-common/src/test/java/org/apache/hadoop/net/TestScriptBasedMapping.java b/hadoop-common/src/test/java/org/apache/hadoop/net/TestScriptBasedMapping.java index b05be5ab464..fc4a781cb5e 100644 --- a/hadoop-common/src/test/java/org/apache/hadoop/net/TestScriptBasedMapping.java +++ b/hadoop-common/src/test/java/org/apache/hadoop/net/TestScriptBasedMapping.java @@ -19,7 +19,6 @@ package org.apache.hadoop.net; import java.util.ArrayList; import java.util.List; -import java.net.UnknownHostException; import org.apache.hadoop.conf.Configuration; @@ -49,37 +48,5 @@ public class TestScriptBasedMapping extends TestCase { List result = mapping.resolve(names); assertNull(result); } - - public void testResolveValidInvalidHostException() { - names = new ArrayList(); - names.add("1.com"); // Add invalid hostname that doesn't resolve - boolean exceptionThrown = false; - try { - mapping.resolveValidHosts(names); - } catch (UnknownHostException e) { - exceptionThrown = true; - } - assertTrue( - "resolveValidHosts did not throw UnknownHostException for invalid host", - exceptionThrown); - } - public void testResolveValidHostNoException() { - conf.setInt(ScriptBasedMapping.SCRIPT_ARG_COUNT_KEY, - ScriptBasedMapping.MIN_ALLOWABLE_ARGS); - conf.set(ScriptBasedMapping.SCRIPT_FILENAME_KEY, "echo"); - mapping.setConf(conf); - - names = new ArrayList(); - names.add("some.machine.name"); - names.add("other.machine.name"); - - boolean exceptionThrown = false; - try { - mapping.resolveValidHosts(names); - } catch (UnknownHostException e) { - exceptionThrown = true; - } - assertFalse("resolveValidHosts threw Exception for valid host", exceptionThrown); - } } diff --git a/hadoop-common/src/test/java/org/apache/hadoop/util/TestDataChecksum.java b/hadoop-common/src/test/java/org/apache/hadoop/util/TestDataChecksum.java index d58fa42f6fa..9f0aba173c1 100644 --- a/hadoop-common/src/test/java/org/apache/hadoop/util/TestDataChecksum.java +++ b/hadoop-common/src/test/java/org/apache/hadoop/util/TestDataChecksum.java @@ -111,6 +111,7 @@ public class TestDataChecksum { } catch (ChecksumException ce) { int expectedPos = checksum.getBytesPerChecksum() * (numSums - 1); assertEquals(expectedPos, ce.getPos()); + assertTrue(ce.getMessage().contains("fake file")); } } diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 753dbe35088..3c8cb5c42a0 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -441,6 +441,11 @@ maven-source-plugin 2.1.2 + + org.apache.maven.plugins + maven-deploy-plugin + 2.5 + @@ -457,9 +462,6 @@ 1.6 - - unix - @@ -502,7 +504,7 @@ os.linux - Linux + !Mac diff --git a/hdfs/CHANGES.txt b/hdfs/CHANGES.txt index 01fbbc7ad52..f6bb3de04d4 100644 --- a/hdfs/CHANGES.txt +++ b/hdfs/CHANGES.txt @@ -629,6 +629,42 @@ Trunk (unreleased changes) HDFS-2199. Move blockTokenSecretManager from FSNamesystem to BlockManager. (Uma Maheswara Rao G via szetszwo) + HDFS-2187. Make EditLogInputStream act like an iterator over FSEditLogOps + (Ivan Kelly and todd via todd) + + HDFS-2225. Refactor edit log file management so it's not in classes + which should be generic to the type of edit log storage. (Ivan Kelly + via todd) + + HDFS-2108. Move datanode heartbeat handling from namenode package to + blockmanagement package. (szetszwo) + + HDFS-2226. Clean up counting of operations in FSEditLogLoader (todd) + + HDFS-2228. Move block and datanode code from FSNamesystem to + BlockManager and DatanodeManager. (szetszwo) + + HDFS-2238. In NamenodeFsck.toString(), uses StringBuilder.(..) instead of + string concatenation. (Uma Maheswara Rao G via szetszwo) + + HDFS-2230. ivy to resolve/retrieve latest common-tests jar published by + hadoop common maven build. (gkesavan) + + HDFS-2227. getRemoteEditLogManifest should pull its information from + FileJournalManager during checkpoint process (Ivan Kelly and Todd Lipcon + via todd) + + HDFS-2239. Reduce access levels of the fields and methods in FSNamesystem. + (szetszwo) + + HDFS-2241. Remove implementing FSConstants interface to just get the + constants from the interface. (suresh) + + HDFS-2237. Change UnderReplicatedBlocks from public to package private. + (szetszwo) + + HDFS-2233. Add WebUI tests with URI reserved chars. (eli) + OPTIMIZATIONS HDFS-1458. Improve checkpoint performance by avoiding unnecessary image @@ -917,6 +953,22 @@ Trunk (unreleased changes) HDFS-2196. Make ant build system work with hadoop-common JAR generated by Maven. (Alejandro Abdelnur via tomwhite) + HDFS-2245. Fix a NullPointerException in BlockManager.chooseTarget(..). + (szetszwo) + + HDFS-2229. Fix a deadlock in namenode by enforcing lock acquisition + ordering. (szetszwo) + + HDFS-2235. Encode servlet paths. (eli) + + HDFS-2186. DN volume failures on startup are not counted. (eli) + + HDFS-2240. Fix a deadlock in LeaseRenewer by enforcing lock acquisition + ordering. (szetszwo) + + HDFS-73. DFSOutputStream does not close all the sockets. + (Uma Maheswara Rao G via eli) + BREAKDOWN OF HDFS-1073 SUBTASKS HDFS-1521. Persist transaction ID on disk between NN restarts. diff --git a/hdfs/build.xml b/hdfs/build.xml index 7dd317bf85e..25927fdb747 100644 --- a/hdfs/build.xml +++ b/hdfs/build.xml @@ -180,7 +180,7 @@ - + @@ -247,8 +247,8 @@ - - + + diff --git a/hdfs/ivy.xml b/hdfs/ivy.xml index 2a67b39e1ab..da70f27b3ec 100644 --- a/hdfs/ivy.xml +++ b/hdfs/ivy.xml @@ -15,7 +15,7 @@ limitations under the License. --> - + @@ -87,7 +87,9 @@ - + + + diff --git a/hdfs/ivy/ivysettings.xml b/hdfs/ivy/ivysettings.xml index 0e8b8253fba..a72aafc10db 100644 --- a/hdfs/ivy/ivysettings.xml +++ b/hdfs/ivy/ivysettings.xml @@ -31,7 +31,8 @@ --> - + + @@ -41,10 +42,11 @@ + checkmodified="true" changingPattern=".*SNAPSHOT" /> + - + diff --git a/hdfs/ivy/libraries.properties b/hdfs/ivy/libraries.properties index 5468d3780b4..ccb4431d264 100644 --- a/hdfs/ivy/libraries.properties +++ b/hdfs/ivy/libraries.properties @@ -41,7 +41,7 @@ hadoop-hdfs.version=0.23.0-SNAPSHOT hsqldb.version=1.8.0.10 -ivy.version=2.1.0 +ivy.version=2.2.0-rc1 jasper.version=5.5.12 jdeb.version=0.8 diff --git a/hdfs/src/contrib/build-contrib.xml b/hdfs/src/contrib/build-contrib.xml index 0c135ce4660..0c57fb90a21 100644 --- a/hdfs/src/contrib/build-contrib.xml +++ b/hdfs/src/contrib/build-contrib.xml @@ -82,7 +82,7 @@ + value="${ant.project.name}/[conf]/[artifact]-[revision](-[classifier]).[ext]"/> diff --git a/hdfs/src/contrib/fuse-dfs/ivy.xml b/hdfs/src/contrib/fuse-dfs/ivy.xml index ef16eeaab0b..c99e16eb6d7 100644 --- a/hdfs/src/contrib/fuse-dfs/ivy.xml +++ b/hdfs/src/contrib/fuse-dfs/ivy.xml @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. --> - + @@ -45,13 +45,20 @@ rev="${hadoop-common.version}" conf="common->default"/> + conf="common->default"> + + + conf="common->master"> + + + + + 0) { // make sure only check once - ExpWarnDays = 0; - long expTimeThreshold = warnDays * MM_SECONDS_PER_DAY - + System.currentTimeMillis(); - X509Certificate[] clientCerts = (X509Certificate[]) conn - .getLocalCertificates(); - if (clientCerts != null) { - for (X509Certificate cert : clientCerts) { - long expTime = cert.getNotAfter().getTime(); - if (expTime < expTimeThreshold) { - StringBuilder sb = new StringBuilder(); - sb.append("\n Client certificate " - + cert.getSubjectX500Principal().getName()); - int dayOffSet = (int) ((expTime - System.currentTimeMillis()) / MM_SECONDS_PER_DAY); - sb.append(" have " + dayOffSet + " days to expire"); - LOG.warn(sb.toString()); - } + // check cert expiration date + final int warnDays = ExpWarnDays; + if (warnDays > 0) { // make sure only check once + ExpWarnDays = 0; + long expTimeThreshold = warnDays * MM_SECONDS_PER_DAY + + System.currentTimeMillis(); + X509Certificate[] clientCerts = (X509Certificate[]) conn + .getLocalCertificates(); + if (clientCerts != null) { + for (X509Certificate cert : clientCerts) { + long expTime = cert.getNotAfter().getTime(); + if (expTime < expTimeThreshold) { + StringBuilder sb = new StringBuilder(); + sb.append("\n Client certificate " + + cert.getSubjectX500Principal().getName()); + int dayOffSet = (int) ((expTime - System.currentTimeMillis()) / MM_SECONDS_PER_DAY); + sb.append(" have " + dayOffSet + " days to expire"); + LOG.warn(sb.toString()); } } } - return (HttpURLConnection) conn; - } catch (URISyntaxException e) { - throw (IOException) new IOException().initCause(e); } + return (HttpURLConnection) conn; } @Override diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/LeaseRenewer.java b/hdfs/src/java/org/apache/hadoop/hdfs/LeaseRenewer.java index e86ac1e68c2..ba26ad2c249 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/LeaseRenewer.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/LeaseRenewer.java @@ -75,7 +75,9 @@ class LeaseRenewer { /** Get a {@link LeaseRenewer} instance */ static LeaseRenewer getInstance(final String authority, final UserGroupInformation ugi, final DFSClient dfsc) throws IOException { - return Factory.INSTANCE.get(authority, ugi, dfsc); + final LeaseRenewer r = Factory.INSTANCE.get(authority, ugi); + r.addClient(dfsc); + return r; } /** @@ -132,14 +134,13 @@ class LeaseRenewer { /** Get a renewer. */ private synchronized LeaseRenewer get(final String authority, - final UserGroupInformation ugi, final DFSClient dfsc) { + final UserGroupInformation ugi) { final Key k = new Key(authority, ugi); LeaseRenewer r = renewers.get(k); if (r == null) { r = new LeaseRenewer(k); renewers.put(k, r); } - r.addClient(dfsc); return r; } @@ -196,7 +197,7 @@ class LeaseRenewer { private LeaseRenewer(Factory.Key factorykey) { this.factorykey = factorykey; - setGraceSleepPeriod(LEASE_RENEWER_GRACE_DEFAULT); + unsyncSetGraceSleepPeriod(LEASE_RENEWER_GRACE_DEFAULT); if (LOG.isTraceEnabled()) { instantiationTrace = StringUtils.stringifyException( @@ -251,6 +252,10 @@ class LeaseRenewer { /** Set the grace period and adjust the sleep period accordingly. */ synchronized void setGraceSleepPeriod(final long gracePeriod) { + unsyncSetGraceSleepPeriod(gracePeriod); + } + + private void unsyncSetGraceSleepPeriod(final long gracePeriod) { if (gracePeriod < 100L) { throw new HadoopIllegalArgumentException(gracePeriod + " = gracePeriod < 100ms is too small."); diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hdfs/src/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index ac4a292ef91..4153110ca9f 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -565,7 +565,6 @@ public interface ClientProtocol extends VersionedProtocol { *
  • [3] contains number of under replicated blocks in the system.
    • *
  • [4] contains number of blocks with a corrupt replica.
    • *
  • [5] contains number of blocks without any good replicas left.
    • - *
  • [5] contains number of blocks without any good replicas left.
    • *
  • [6] contains the total used space of the block pool.
    • * * Use public constants like {@link #GET_STATS_CAPACITY_IDX} in place of diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java b/hdfs/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java index 5e803f2282b..f023d73c932 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java @@ -18,68 +18,71 @@ package org.apache.hadoop.hdfs.protocol; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; /************************************ * Some handy constants - * + * ************************************/ @InterfaceAudience.Private -public interface FSConstants { +public final class FSConstants { + /* Hidden constructor */ + private FSConstants() { + } + public static int MIN_BLOCKS_FOR_WRITE = 5; // Long that indicates "leave current quota unchanged" public static final long QUOTA_DONT_SET = Long.MAX_VALUE; public static final long QUOTA_RESET = -1L; - + // // Timeouts, constants // - public static long HEARTBEAT_INTERVAL = 3; - public static long BLOCKREPORT_INTERVAL = 60 * 60 * 1000; - public static long BLOCKREPORT_INITIAL_DELAY = 0; public static final long LEASE_SOFTLIMIT_PERIOD = 60 * 1000; public static final long LEASE_HARDLIMIT_PERIOD = 60 * LEASE_SOFTLIMIT_PERIOD; - public static final long LEASE_RECOVER_PERIOD = 10 * 1000; //in ms - - // We need to limit the length and depth of a path in the filesystem. HADOOP-438 - // Currently we set the maximum length to 8k characters and the maximum depth to 1k. + public static final long LEASE_RECOVER_PERIOD = 10 * 1000; // in ms + + // We need to limit the length and depth of a path in the filesystem. + // HADOOP-438 + // Currently we set the maximum length to 8k characters and the maximum depth + // to 1k. public static int MAX_PATH_LENGTH = 8000; public static int MAX_PATH_DEPTH = 1000; - - public static final int BUFFER_SIZE = new HdfsConfiguration().getInt("io.file.buffer.size", 4096); - //Used for writing header etc. - public static final int SMALL_BUFFER_SIZE = Math.min(BUFFER_SIZE/2, 512); - //TODO mb@media-style.com: should be conf injected? - public static final long DEFAULT_BLOCK_SIZE = 64 * 1024 * 1024; - public static final int DEFAULT_BYTES_PER_CHECKSUM = 512; - public static final int DEFAULT_WRITE_PACKET_SIZE = 64 * 1024; - public static final short DEFAULT_REPLICATION_FACTOR = 3; - public static final int DEFAULT_FILE_BUFFER_SIZE = 4096; - public static final int DEFAULT_DATA_SOCKET_SIZE = 128 * 1024; - public static final int SIZE_OF_INTEGER = Integer.SIZE / Byte.SIZE; + // TODO mb@media-style.com: should be conf injected? + public static final int DEFAULT_DATA_SOCKET_SIZE = 128 * 1024; + public static final int IO_FILE_BUFFER_SIZE = new HdfsConfiguration().getInt( + DFSConfigKeys.IO_FILE_BUFFER_SIZE_KEY, + DFSConfigKeys.IO_FILE_BUFFER_SIZE_DEFAULT); + // Used for writing header etc. + public static final int SMALL_BUFFER_SIZE = Math.min(IO_FILE_BUFFER_SIZE / 2, + 512); + + public static final int BYTES_IN_INTEGER = Integer.SIZE / Byte.SIZE; // SafeMode actions - public enum SafeModeAction{ SAFEMODE_LEAVE, SAFEMODE_ENTER, SAFEMODE_GET; } + public enum SafeModeAction { + SAFEMODE_LEAVE, SAFEMODE_ENTER, SAFEMODE_GET; + } // type of the datanode report - public static enum DatanodeReportType {ALL, LIVE, DEAD } - + public static enum DatanodeReportType { + ALL, LIVE, DEAD + } + // An invalid transaction ID that will never be seen in a real namesystem. public static final long INVALID_TXID = -12345; /** * Distributed upgrade actions: * - * 1. Get upgrade status. - * 2. Get detailed upgrade status. - * 3. Proceed with the upgrade if it is stuck, no matter what the status is. + * 1. Get upgrade status. 2. Get detailed upgrade status. 3. Proceed with the + * upgrade if it is stuck, no matter what the status is. */ public static enum UpgradeAction { - GET_STATUS, - DETAILED_STATUS, - FORCE_PROCEED; + GET_STATUS, DETAILED_STATUS, FORCE_PROCEED; } /** @@ -90,6 +93,6 @@ public interface FSConstants { /** * Please see {@link LayoutVersion} on adding new layout version. */ - public static final int LAYOUT_VERSION = - LayoutVersion.getCurrentLayoutVersion(); + public static final int LAYOUT_VERSION = LayoutVersion + .getCurrentLayoutVersion(); } diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java index 5bc4c0c75e7..260cd7600b8 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java @@ -309,10 +309,10 @@ public class Balancer { target.datanode.getName()), HdfsConstants.READ_TIMEOUT); sock.setKeepAlive(true); out = new DataOutputStream( new BufferedOutputStream( - sock.getOutputStream(), FSConstants.BUFFER_SIZE)); + sock.getOutputStream(), FSConstants.IO_FILE_BUFFER_SIZE)); sendRequest(out); in = new DataInputStream( new BufferedInputStream( - sock.getInputStream(), FSConstants.BUFFER_SIZE)); + sock.getInputStream(), FSConstants.IO_FILE_BUFFER_SIZE)); receiveResponse(in); bytesMoved.inc(block.getNumBytes()); LOG.info( "Moving block " + block.getBlock().getBlockId() + diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java index 5f10f2ae754..e7c160af251 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java @@ -259,26 +259,6 @@ public class BlockInfo extends Block implements LightWeightGSet.LinkedElement { return head; } - boolean listIsConsistent(DatanodeDescriptor dn) { - // going forward - int count = 0; - BlockInfo next, nextPrev; - BlockInfo cur = this; - while(cur != null) { - next = cur.getNext(cur.findDatanode(dn)); - if(next != null) { - nextPrev = next.getPrevious(next.findDatanode(dn)); - if(cur != nextPrev) { - System.out.println("Inconsistent list: cur->next->prev != cur"); - return false; - } - } - cur = next; - count++; - } - return true; - } - /** * BlockInfo represents a block that is not being constructed. * In order to start modifying the block, the BlockInfo should be converted diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index c15f9fe3844..19e604c3a73 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -17,8 +17,6 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import static org.apache.hadoop.hdfs.server.common.Util.now; - import java.io.IOException; import java.io.PrintWriter; import java.util.ArrayList; @@ -46,14 +44,16 @@ import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs.BlockReportIterator; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException; -import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager; +import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager.AccessMode; import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; -import org.apache.hadoop.hdfs.server.blockmanagement.UnderReplicatedBlocks.BlockIterator; import org.apache.hadoop.hdfs.server.common.HdfsConstants.BlockUCState; import org.apache.hadoop.hdfs.server.common.HdfsConstants.ReplicaState; +import org.apache.hadoop.hdfs.server.common.Util; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INodeFile; @@ -61,8 +61,9 @@ import org.apache.hadoop.hdfs.server.namenode.INodeFileUnderConstruction; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations; import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations; +import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand; +import org.apache.hadoop.hdfs.server.protocol.KeyUpdateCommand; import org.apache.hadoop.net.Node; -import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.Daemon; /** @@ -82,23 +83,13 @@ public class BlockManager { private volatile long pendingReplicationBlocksCount = 0L; private volatile long corruptReplicaBlocksCount = 0L; private volatile long underReplicatedBlocksCount = 0L; - public volatile long scheduledReplicationBlocksCount = 0L; + private volatile long scheduledReplicationBlocksCount = 0L; private volatile long excessBlocksCount = 0L; private volatile long pendingDeletionBlocksCount = 0L; private boolean isBlockTokenEnabled; private long blockKeyUpdateInterval; private long blockTokenLifetime; private BlockTokenSecretManager blockTokenSecretManager; - - /** returns the isBlockTokenEnabled - true if block token enabled ,else false */ - public boolean isBlockTokenEnabled() { - return isBlockTokenEnabled; - } - - /** get the block key update interval */ - public long getBlockKeyUpdateInterval() { - return blockKeyUpdateInterval; - } /** get the BlockTokenSecretManager */ public BlockTokenSecretManager getBlockTokenSecretManager() { @@ -137,10 +128,11 @@ public class BlockManager { * Mapping: Block -> { INode, datanodes, self ref } * Updated only in response to client-sent information. */ - public final BlocksMap blocksMap; + final BlocksMap blocksMap; private final DatanodeManager datanodeManager; - + private final HeartbeatManager heartbeatManager; + /** Replication thread. */ final Daemon replicationThread = new Daemon(new ReplicationMonitor()); @@ -173,13 +165,13 @@ public class BlockManager { private final PendingReplicationBlocks pendingReplications; /** The maximum number of replicas allowed for a block */ - public final int maxReplication; + public final short maxReplication; /** The maximum number of outgoing replication streams * a given node should have at one time */ - public int maxReplicationStreams; + int maxReplicationStreams; /** Minimum copies needed or else write is disallowed */ - public final int minReplication; + public final short minReplication; /** Default number of replicas */ public final int defaultReplication; /** The maximum number of entries returned by getCorruptInodes() */ @@ -194,45 +186,11 @@ public class BlockManager { /** for block replicas placement */ private BlockPlacementPolicy blockplacement; - /** - * Get access keys - * - * @return current access keys - */ - public ExportedBlockKeys getBlockKeys() { - return isBlockTokenEnabled ? blockTokenSecretManager.exportKeys() - : ExportedBlockKeys.DUMMY_KEYS; - } - - /** Generate block token for a LocatedBlock. */ - public void setBlockToken(LocatedBlock l) throws IOException { - Token token = blockTokenSecretManager.generateToken(l - .getBlock(), EnumSet.of(BlockTokenSecretManager.AccessMode.READ)); - l.setBlockToken(token); - } - - /** Generate block tokens for the blocks to be returned. */ - public void setBlockTokens(List locatedBlocks) throws IOException { - for(LocatedBlock l : locatedBlocks) { - setBlockToken(l); - } - } - - /** - * Update access keys. - */ - public void updateBlockKey() throws IOException { - this.blockTokenSecretManager.updateKeys(); - synchronized (namesystem.heartbeats) { - for (DatanodeDescriptor nodeInfo : namesystem.heartbeats) { - nodeInfo.needKeyUpdate = true; - } - } - } - public BlockManager(FSNamesystem fsn, Configuration conf) throws IOException { namesystem = fsn; - datanodeManager = new DatanodeManager(fsn, conf); + datanodeManager = new DatanodeManager(this, fsn, conf); + heartbeatManager = datanodeManager.getHeartbeatManager(); + blocksMap = new BlocksMap(DEFAULT_MAP_LOAD_FACTOR); blockplacement = BlockPlacementPolicy.getInstance( conf, namesystem, datanodeManager.getNetworkTopology()); @@ -264,25 +222,28 @@ public class BlockManager { DFSConfigKeys.DFS_DEFAULT_MAX_CORRUPT_FILES_RETURNED); this.defaultReplication = conf.getInt(DFSConfigKeys.DFS_REPLICATION_KEY, DFSConfigKeys.DFS_REPLICATION_DEFAULT); - this.maxReplication = conf.getInt(DFSConfigKeys.DFS_REPLICATION_MAX_KEY, - DFSConfigKeys.DFS_REPLICATION_MAX_DEFAULT); - this.minReplication = conf.getInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY, - DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_DEFAULT); - if (minReplication <= 0) - throw new IOException( - "Unexpected configuration parameters: dfs.namenode.replication.min = " - + minReplication - + " must be greater than 0"); - if (maxReplication >= (int)Short.MAX_VALUE) - throw new IOException( - "Unexpected configuration parameters: dfs.replication.max = " - + maxReplication + " must be less than " + (Short.MAX_VALUE)); - if (maxReplication < minReplication) - throw new IOException( - "Unexpected configuration parameters: dfs.namenode.replication.min = " - + minReplication - + " must be less than dfs.replication.max = " - + maxReplication); + + final int maxR = conf.getInt(DFSConfigKeys.DFS_REPLICATION_MAX_KEY, + DFSConfigKeys.DFS_REPLICATION_MAX_DEFAULT); + final int minR = conf.getInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY, + DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_DEFAULT); + if (minR <= 0) + throw new IOException("Unexpected configuration parameters: " + + DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY + + " = " + minR + " <= 0"); + if (maxR > Short.MAX_VALUE) + throw new IOException("Unexpected configuration parameters: " + + DFSConfigKeys.DFS_REPLICATION_MAX_KEY + + " = " + maxR + " > " + Short.MAX_VALUE); + if (minR > maxR) + throw new IOException("Unexpected configuration parameters: " + + DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY + + " = " + minR + " > " + + DFSConfigKeys.DFS_REPLICATION_MAX_KEY + + " = " + maxR); + this.minReplication = (short)minR; + this.maxReplication = (short)maxR; + this.maxReplicationStreams = conf.getInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_DEFAULT); this.shouldCheckForEnoughRacks = conf.get(DFSConfigKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY) == null ? false @@ -387,6 +348,11 @@ public class BlockManager { getDatanodeManager().datanodeDump(out); } + /** @return maxReplicationStreams */ + public int getMaxReplicationStreams() { + return maxReplicationStreams; + } + /** * @param block * @return true if the block has minimum replicas @@ -413,19 +379,8 @@ public class BlockManager { "commitBlock length is less than the stored one " + commitBlock.getNumBytes() + " vs. " + block.getNumBytes(); block.commitBlock(commitBlock); - - // Adjust disk space consumption if required - long diff = fileINode.getPreferredBlockSize() - commitBlock.getNumBytes(); - if (diff > 0) { - try { - String path = /* For finding parents */ - namesystem.leaseManager.findPath(fileINode); - namesystem.dir.updateSpaceConsumed(path, 0, -diff - * fileINode.getReplication()); - } catch (IOException e) { - LOG.warn("Unexpected exception while updating disk space.", e); - } - } + + namesystem.updateDiskSpaceConsumed(fileINode, commitBlock); } /** @@ -527,7 +482,7 @@ public class BlockManager { } long fileLength = fileINode.computeContentSummary().getLength(); - return getBlockLocation(ucBlock, fileLength - ucBlock.getNumBytes()); + return createLocatedBlock(ucBlock, fileLength - ucBlock.getNumBytes()); } /** @@ -547,8 +502,9 @@ public class BlockManager { return machineSet; } - public List getBlockLocations(BlockInfo[] blocks, long offset, - long length, int nrBlocksToReturn) throws IOException { + private List createLocatedBlockList(final BlockInfo[] blocks, + final long offset, final long length, final int nrBlocksToReturn + ) throws IOException { int curBlk = 0; long curPos = 0, blkSize = 0; int nrBlocks = (blocks[0].getNumBytes() == 0) ? 0 : blocks.length; @@ -567,7 +523,7 @@ public class BlockManager { long endOff = offset + length; List results = new ArrayList(blocks.length); do { - results.add(getBlockLocation(blocks[curBlk], curPos)); + results.add(createLocatedBlock(blocks[curBlk], curPos)); curPos += blocks[curBlk].getNumBytes(); curBlk++; } while (curPos < endOff @@ -577,7 +533,7 @@ public class BlockManager { } /** @return a LocatedBlock for the given block */ - public LocatedBlock getBlockLocation(final BlockInfo blk, final long pos + private LocatedBlock createLocatedBlock(final BlockInfo blk, final long pos ) throws IOException { if (blk instanceof BlockInfoUnderConstruction) { if (blk.isComplete()) { @@ -587,7 +543,8 @@ public class BlockManager { } final BlockInfoUnderConstruction uc = (BlockInfoUnderConstruction)blk; final DatanodeDescriptor[] locations = uc.getExpectedLocations(); - return namesystem.createLocatedBlock(uc, locations, pos, false); + final ExtendedBlock eb = new ExtendedBlock(namesystem.getBlockPoolId(), blk); + return new LocatedBlock(eb, locations, pos, false); } // get block locations @@ -613,7 +570,78 @@ public class BlockManager { machines[j++] = d; } } - return namesystem.createLocatedBlock(blk, machines, pos, isCorrupt); + final ExtendedBlock eb = new ExtendedBlock(namesystem.getBlockPoolId(), blk); + return new LocatedBlock(eb, machines, pos, isCorrupt); + } + + /** Create a LocatedBlocks. */ + public LocatedBlocks createLocatedBlocks(final BlockInfo[] blocks, + final long fileSizeExcludeBlocksUnderConstruction, + final boolean isFileUnderConstruction, + final long offset, final long length, final boolean needBlockToken + ) throws IOException { + assert namesystem.hasReadOrWriteLock(); + if (blocks == null) { + return null; + } else if (blocks.length == 0) { + return new LocatedBlocks(0, isFileUnderConstruction, + Collections.emptyList(), null, false); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("blocks = " + java.util.Arrays.asList(blocks)); + } + final List locatedblocks = createLocatedBlockList( + blocks, offset, length, Integer.MAX_VALUE); + + final BlockInfo last = blocks[blocks.length - 1]; + final long lastPos = last.isComplete()? + fileSizeExcludeBlocksUnderConstruction - last.getNumBytes() + : fileSizeExcludeBlocksUnderConstruction; + final LocatedBlock lastlb = createLocatedBlock(last, lastPos); + + if (isBlockTokenEnabled && needBlockToken) { + for(LocatedBlock lb : locatedblocks) { + setBlockToken(lb, AccessMode.READ); + } + setBlockToken(lastlb, AccessMode.READ); + } + return new LocatedBlocks( + fileSizeExcludeBlocksUnderConstruction, isFileUnderConstruction, + locatedblocks, lastlb, last.isComplete()); + } + } + + /** @return current access keys. */ + public ExportedBlockKeys getBlockKeys() { + return isBlockTokenEnabled? blockTokenSecretManager.exportKeys() + : ExportedBlockKeys.DUMMY_KEYS; + } + + /** Generate a block token for the located block. */ + public void setBlockToken(final LocatedBlock b, + final BlockTokenSecretManager.AccessMode mode) throws IOException { + if (isBlockTokenEnabled) { + b.setBlockToken(blockTokenSecretManager.generateToken(b.getBlock(), + EnumSet.of(mode))); + } + } + + void addKeyUpdateCommand(final List cmds, + final DatanodeDescriptor nodeinfo) { + // check access key update + if (isBlockTokenEnabled && nodeinfo.needKeyUpdate) { + cmds.add(new KeyUpdateCommand(blockTokenSecretManager.exportKeys())); + nodeinfo.needKeyUpdate = false; + } + } + + /** + * Clamp the specified replication between the minimum and the maximum + * replication levels. + */ + public short adjustReplication(short replication) { + return replication < minReplication? minReplication + : replication > maxReplication? maxReplication: replication; } /** @@ -642,12 +670,30 @@ public class BlockManager { minReplication); } - /** Get all blocks with location information from a datanode. */ - public BlocksWithLocations getBlocksWithLocations(final DatanodeID datanode, + /** + * return a list of blocks & their locations on datanode whose + * total size is size + * + * @param datanode on which blocks are located + * @param size total size of blocks + */ + public BlocksWithLocations getBlocks(DatanodeID datanode, long size + ) throws IOException { + namesystem.readLock(); + try { + namesystem.checkSuperuserPrivilege(); + return getBlocksWithLocations(datanode, size); + } finally { + namesystem.readUnlock(); + } + } + + /** Get all blocks with location information from a datanode. */ + private BlocksWithLocations getBlocksWithLocations(final DatanodeID datanode, final long size) throws UnregisteredNodeException { final DatanodeDescriptor node = getDatanodeManager().getDatanode(datanode); if (node == null) { - NameNode.stateChangeLog.warn("BLOCK* NameSystem.getBlocks: " + NameNode.stateChangeLog.warn("BLOCK* getBlocks: " + "Asking for blocks from an unrecorded node " + datanode.getName()); throw new HadoopIllegalArgumentException( "Datanode " + datanode.getName() + " not found."); @@ -685,8 +731,8 @@ public class BlockManager { } - /** Remove a datanode. */ - public void removeDatanode(final DatanodeDescriptor node) { + /** Remove the blocks associated to the given datanode. */ + void removeBlocksAssociatedTo(final DatanodeDescriptor node) { final Iterator it = node.getBlockIterator(); while(it.hasNext()) { removeStoredBlock(it.next(), node); @@ -694,11 +740,6 @@ public class BlockManager { node.resetBlocks(); removeFromInvalidates(node.getStorageID()); - datanodeManager.getNetworkTopology().remove(node); - - if (LOG.isDebugEnabled()) { - LOG.debug("remove datanode " + node.getName()); - } } private void removeFromInvalidates(String storageID, Block block) { @@ -724,7 +765,7 @@ public class BlockManager { * @param dn datanode * @param log true to create an entry in the log */ - void addToInvalidates(Block b, DatanodeInfo dn, boolean log) { + private void addToInvalidates(Block b, DatanodeInfo dn, boolean log) { Collection invalidateSet = recentInvalidateSets .get(dn.getStorageID()); if (invalidateSet == null) { @@ -734,7 +775,7 @@ public class BlockManager { if (invalidateSet.add(b)) { pendingDeletionBlocksCount++; if (log) { - NameNode.stateChangeLog.info("BLOCK* NameSystem.addToInvalidates: " + NameNode.stateChangeLog.info("BLOCK* addToInvalidates: " + b + " to " + dn.getName()); } } @@ -747,7 +788,7 @@ public class BlockManager { * @param b block * @param dn datanode */ - public void addToInvalidates(Block b, DatanodeInfo dn) { + void addToInvalidates(Block b, DatanodeInfo dn) { addToInvalidates(b, dn, true); } @@ -764,7 +805,7 @@ public class BlockManager { datanodes.append(node.getName()).append(" "); } if (datanodes.length() != 0) { - NameNode.stateChangeLog.info("BLOCK* NameSystem.addToInvalidates: " + NameNode.stateChangeLog.info("BLOCK* addToInvalidates: " + b + " to " + datanodes.toString()); } } @@ -788,20 +829,29 @@ public class BlockManager { } } - public void findAndMarkBlockAsCorrupt(Block blk, - DatanodeInfo dn) throws IOException { - BlockInfo storedBlock = getStoredBlock(blk); - if (storedBlock == null) { - // Check if the replica is in the blockMap, if not - // ignore the request for now. This could happen when BlockScanner - // thread of Datanode reports bad block before Block reports are sent - // by the Datanode on startup - NameNode.stateChangeLog.info("BLOCK* NameSystem.markBlockAsCorrupt: " + - "block " + blk + " could not be marked as " + - "corrupt as it does not exist in blocksMap"); - return; + /** + * Mark the block belonging to datanode as corrupt + * @param blk Block to be marked as corrupt + * @param dn Datanode which holds the corrupt replica + */ + public void findAndMarkBlockAsCorrupt(final ExtendedBlock blk, + final DatanodeInfo dn) throws IOException { + namesystem.writeLock(); + try { + final BlockInfo storedBlock = getStoredBlock(blk.getLocalBlock()); + if (storedBlock == null) { + // Check if the replica is in the blockMap, if not + // ignore the request for now. This could happen when BlockScanner + // thread of Datanode reports bad block before Block reports are sent + // by the Datanode on startup + NameNode.stateChangeLog.info("BLOCK* findAndMarkBlockAsCorrupt: " + + blk + " not found."); + return; + } + markBlockAsCorrupt(storedBlock, dn); + } finally { + namesystem.writeUnlock(); } - markBlockAsCorrupt(storedBlock, dn); } private void markBlockAsCorrupt(BlockInfo storedBlock, @@ -817,7 +867,7 @@ public class BlockManager { INodeFile inode = storedBlock.getINode(); if (inode == null) { - NameNode.stateChangeLog.info("BLOCK NameSystem.markBlockAsCorrupt: " + + NameNode.stateChangeLog.info("BLOCK markBlockAsCorrupt: " + "block " + storedBlock + " could not be marked as corrupt as it" + " does not belong to any file"); @@ -844,13 +894,12 @@ public class BlockManager { */ private void invalidateBlock(Block blk, DatanodeInfo dn) throws IOException { - NameNode.stateChangeLog.info("DIR* NameSystem.invalidateBlock: " + NameNode.stateChangeLog.info("BLOCK* invalidateBlock: " + blk + " on " + dn.getName()); DatanodeDescriptor node = getDatanodeManager().getDatanode(dn); if (node == null) { - throw new IOException("Cannot invalidate block " + blk + - " because datanode " + dn.getName() + - " does not exist."); + throw new IOException("Cannot invalidate block " + blk + + " because datanode " + dn.getName() + " does not exist."); } // Check how many copies we have of the block. If we have at least one @@ -860,14 +909,12 @@ public class BlockManager { addToInvalidates(blk, dn); removeStoredBlock(blk, node); if(NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug("BLOCK* NameSystem.invalidateBlocks: " - + blk + " on " - + dn.getName() + " listed for deletion."); + NameNode.stateChangeLog.debug("BLOCK* invalidateBlocks: " + + blk + " on " + dn.getName() + " listed for deletion."); } } else { - NameNode.stateChangeLog.info("BLOCK* NameSystem.invalidateBlocks: " - + blk + " on " + dn.getName() - + " is the only copy and was not deleted."); + NameNode.stateChangeLog.info("BLOCK* invalidateBlocks: " + blk + " on " + + dn.getName() + " is the only copy and was not deleted."); } } @@ -887,7 +934,7 @@ public class BlockManager { * @param nodesToProcess number of datanodes to schedule deletion work * @return total number of block for deletion */ - public int computeInvalidateWork(int nodesToProcess) { + int computeInvalidateWork(int nodesToProcess) { int numOfNodes = recentInvalidateSets.size(); nodesToProcess = Math.min(numOfNodes, nodesToProcess); @@ -927,7 +974,7 @@ public class BlockManager { * * @return number of blocks scheduled for replication during this iteration. */ - public int computeReplicationWork(int blocksToProcess) throws IOException { + private int computeReplicationWork(int blocksToProcess) throws IOException { // Choose the blocks to be replicated List> blocksToReplicate = chooseUnderReplicatedBlocks(blocksToProcess); @@ -1174,12 +1221,13 @@ public class BlockManager { final DatanodeDescriptor targets[] = blockplacement.chooseTarget( src, numOfReplicas, client, excludedNodes, blocksize); if (targets.length < minReplication) { - throw new IOException("File " + src + " could only be replicated to " + - targets.length + " nodes, instead of " + - minReplication + ". There are " - + getDatanodeManager().getNetworkTopology().getNumOfLeaves() - + " datanode(s) running but "+excludedNodes.size() + - " node(s) are excluded in this operation."); + throw new IOException("File " + src + " could only be replicated to " + + targets.length + " nodes instead of minReplication (=" + + minReplication + "). There are " + + getDatanodeManager().getNetworkTopology().getNumOfLeaves() + + " datanode(s) running and " + + (excludedNodes == null? "no": excludedNodes.size()) + + " node(s) are excluded in this operation."); } return targets; } @@ -1299,20 +1347,51 @@ public class BlockManager { } /** - * The given node is reporting all its blocks. Use this info to - * update the (datanode-->blocklist) and (block-->nodelist) tables. + * The given datanode is reporting all its blocks. + * Update the (machine-->blocklist) and (block-->machinelist) maps. */ - public void processReport(DatanodeDescriptor node, BlockListAsLongs report) - throws IOException { - - boolean isFirstBlockReport = (node.numBlocks() == 0); - if (isFirstBlockReport) { - // Initial block reports can be processed a lot more efficiently than - // ordinary block reports. This shortens NN restart times. - processFirstBlockReport(node, report); - return; - } + public void processReport(final DatanodeID nodeID, final String poolId, + final BlockListAsLongs newReport) throws IOException { + namesystem.writeLock(); + final long startTime = Util.now(); //after acquiring write lock + final long endTime; + try { + final DatanodeDescriptor node = datanodeManager.getDatanode(nodeID); + if (node == null || !node.isAlive) { + throw new IOException("ProcessReport from dead or unregistered node: " + + nodeID.getName()); + } + // To minimize startup time, we discard any second (or later) block reports + // that we receive while still in startup phase. + if (namesystem.isInStartupSafeMode() && node.numBlocks() > 0) { + NameNode.stateChangeLog.info("BLOCK* processReport: " + + "discarded non-initial block report from " + nodeID.getName() + + " because namenode still in startup phase"); + return; + } + + if (node.numBlocks() == 0) { + // The first block report can be processed a lot more efficiently than + // ordinary block reports. This shortens restart times. + processFirstBlockReport(node, newReport); + } else { + processReport(node, newReport); + } + } finally { + endTime = Util.now(); + namesystem.writeUnlock(); + } + + // Log the block report processing stats from Namenode perspective + NameNode.getNameNodeMetrics().addBlockReport((int) (endTime - startTime)); + NameNode.stateChangeLog.info("BLOCK* processReport: from " + + nodeID.getName() + ", blocks: " + newReport.getNumberOfBlocks() + + ", processing time: " + (endTime - startTime) + " msecs"); + } + + private void processReport(final DatanodeDescriptor node, + final BlockListAsLongs report) throws IOException { // Normal case: // Modify the (block-->datanode) map, according to the difference // between the old and new block report. @@ -1335,7 +1414,7 @@ public class BlockManager { addStoredBlock(b, node, null, true); } for (Block b : toInvalidate) { - NameNode.stateChangeLog.info("BLOCK* NameSystem.processReport: block " + NameNode.stateChangeLog.info("BLOCK* processReport: block " + b + " on " + node.getName() + " size " + b.getNumBytes() + " does not belong to any file."); addToInvalidates(b, node); @@ -1356,8 +1435,8 @@ public class BlockManager { * @param report - the initial block report, to be processed * @throws IOException */ - void processFirstBlockReport(DatanodeDescriptor node, BlockListAsLongs report) - throws IOException { + private void processFirstBlockReport(final DatanodeDescriptor node, + final BlockListAsLongs report) throws IOException { if (report == null) return; assert (namesystem.hasWriteLock()); assert (node.numBlocks() == 0); @@ -1454,12 +1533,12 @@ public class BlockManager { * @param toUC replicas of blocks currently under construction * @return */ - BlockInfo processReportedBlock(DatanodeDescriptor dn, - Block block, ReplicaState reportedState, - Collection toAdd, - Collection toInvalidate, - Collection toCorrupt, - Collection toUC) { + private BlockInfo processReportedBlock(final DatanodeDescriptor dn, + final Block block, final ReplicaState reportedState, + final Collection toAdd, + final Collection toInvalidate, + final Collection toCorrupt, + final Collection toUC) { if(LOG.isDebugEnabled()) { LOG.debug("Reported block " + block @@ -1629,11 +1708,9 @@ public class BlockManager { } if (storedBlock == null || storedBlock.getINode() == null) { // If this block does not belong to anyfile, then we are done. - NameNode.stateChangeLog.info("BLOCK* NameSystem.addStoredBlock: " - + "addStoredBlock request received for " - + block + " on " + node.getName() - + " size " + block.getNumBytes() - + " But it does not belong to any file."); + NameNode.stateChangeLog.info("BLOCK* addStoredBlock: " + block + " on " + + node.getName() + " size " + block.getNumBytes() + + " but it does not belong to any file."); // we could add this block to invalidate set of this datanode. // it will happen in next block report otherwise. return block; @@ -1649,13 +1726,13 @@ public class BlockManager { if (added) { curReplicaDelta = 1; if (logEveryBlock) { - NameNode.stateChangeLog.info("BLOCK* NameSystem.addStoredBlock: " + NameNode.stateChangeLog.info("BLOCK* addStoredBlock: " + "blockMap updated: " + node.getName() + " is added to " + storedBlock + " size " + storedBlock.getNumBytes()); } } else { curReplicaDelta = 0; - NameNode.stateChangeLog.warn("BLOCK* NameSystem.addStoredBlock: " + NameNode.stateChangeLog.warn("BLOCK* addStoredBlock: " + "Redundant addStoredBlock request received for " + storedBlock + " on " + node.getName() + " size " + storedBlock.getNumBytes()); } @@ -1752,52 +1829,76 @@ public class BlockManager { * over or under replicated. Place it into the respective queue. */ public void processMisReplicatedBlocks() { - long nrInvalid = 0, nrOverReplicated = 0, nrUnderReplicated = 0; - namesystem.writeLock(); - try { - neededReplications.clear(); - for (BlockInfo block : blocksMap.getBlocks()) { - INodeFile fileINode = block.getINode(); - if (fileINode == null) { - // block does not belong to any file - nrInvalid++; - addToInvalidates(block); - continue; - } - // calculate current replication - short expectedReplication = fileINode.getReplication(); - NumberReplicas num = countNodes(block); - int numCurrentReplica = num.liveReplicas(); - // add to under-replicated queue if need to be - if (isNeededReplication(block, expectedReplication, numCurrentReplica)) { - if (neededReplications.add(block, numCurrentReplica, num - .decommissionedReplicas(), expectedReplication)) { - nrUnderReplicated++; - } - } + assert namesystem.hasWriteLock(); - if (numCurrentReplica > expectedReplication) { - // over-replicated block - nrOverReplicated++; - processOverReplicatedBlock(block, expectedReplication, null, null); + long nrInvalid = 0, nrOverReplicated = 0, nrUnderReplicated = 0; + neededReplications.clear(); + for (BlockInfo block : blocksMap.getBlocks()) { + INodeFile fileINode = block.getINode(); + if (fileINode == null) { + // block does not belong to any file + nrInvalid++; + addToInvalidates(block); + continue; + } + // calculate current replication + short expectedReplication = fileINode.getReplication(); + NumberReplicas num = countNodes(block); + int numCurrentReplica = num.liveReplicas(); + // add to under-replicated queue if need to be + if (isNeededReplication(block, expectedReplication, numCurrentReplica)) { + if (neededReplications.add(block, numCurrentReplica, num + .decommissionedReplicas(), expectedReplication)) { + nrUnderReplicated++; } } - } finally { - namesystem.writeUnlock(); + + if (numCurrentReplica > expectedReplication) { + // over-replicated block + nrOverReplicated++; + processOverReplicatedBlock(block, expectedReplication, null, null); + } } + LOG.info("Total number of blocks = " + blocksMap.size()); LOG.info("Number of invalid blocks = " + nrInvalid); LOG.info("Number of under-replicated blocks = " + nrUnderReplicated); LOG.info("Number of over-replicated blocks = " + nrOverReplicated); } + /** Set replication for the blocks. */ + public void setReplication(final short oldRepl, final short newRepl, + final String src, final Block... blocks) throws IOException { + if (newRepl == oldRepl) { + return; + } + + // update needReplication priority queues + for(Block b : blocks) { + updateNeededReplications(b, 0, newRepl-oldRepl); + } + + if (oldRepl > newRepl) { + // old replication > the new one; need to remove copies + LOG.info("Decreasing replication from " + oldRepl + " to " + newRepl + + " for " + src); + for(Block b : blocks) { + processOverReplicatedBlock(b, newRepl, null, null); + } + } else { // replication factor is increased + LOG.info("Increasing replication from " + oldRepl + " to " + newRepl + + " for " + src); + } + } + /** * Find how many of the containing nodes are "extra", if any. * If there are any extras, call chooseExcessReplicates() to * mark them in the excessReplicateMap. */ - public void processOverReplicatedBlock(Block block, short replication, - DatanodeDescriptor addedNode, DatanodeDescriptor delNodeHint) { + private void processOverReplicatedBlock(final Block block, + final short replication, final DatanodeDescriptor addedNode, + DatanodeDescriptor delNodeHint) { assert namesystem.hasWriteLock(); if (addedNode == delNodeHint) { delNodeHint = null; @@ -1819,12 +1920,112 @@ public class BlockManager { } } } - namesystem.chooseExcessReplicates(nonExcess, block, replication, + chooseExcessReplicates(nonExcess, block, replication, addedNode, delNodeHint, blockplacement); } - public void addToExcessReplicate(DatanodeInfo dn, Block block) { + /** + * We want "replication" replicates for the block, but we now have too many. + * In this method, copy enough nodes from 'srcNodes' into 'dstNodes' such that: + * + * srcNodes.size() - dstNodes.size() == replication + * + * We pick node that make sure that replicas are spread across racks and + * also try hard to pick one with least free space. + * The algorithm is first to pick a node with least free space from nodes + * that are on a rack holding more than one replicas of the block. + * So removing such a replica won't remove a rack. + * If no such a node is available, + * then pick a node with least free space + */ + private void chooseExcessReplicates(Collection nonExcess, + Block b, short replication, + DatanodeDescriptor addedNode, + DatanodeDescriptor delNodeHint, + BlockPlacementPolicy replicator) { + assert namesystem.hasWriteLock(); + // first form a rack to datanodes map and + INodeFile inode = getINode(b); + final Map> rackMap + = new HashMap>(); + for(final Iterator iter = nonExcess.iterator(); + iter.hasNext(); ) { + final DatanodeDescriptor node = iter.next(); + final String rackName = node.getNetworkLocation(); + List datanodeList = rackMap.get(rackName); + if (datanodeList == null) { + datanodeList = new ArrayList(); + rackMap.put(rackName, datanodeList); + } + datanodeList.add(node); + } + + // split nodes into two sets + // priSet contains nodes on rack with more than one replica + // remains contains the remaining nodes + final List priSet = new ArrayList(); + final List remains = new ArrayList(); + for(List datanodeList : rackMap.values()) { + if (datanodeList.size() == 1 ) { + remains.add(datanodeList.get(0)); + } else { + priSet.addAll(datanodeList); + } + } + + // pick one node to delete that favors the delete hint + // otherwise pick one with least space from priSet if it is not empty + // otherwise one node with least space from remains + boolean firstOne = true; + while (nonExcess.size() - replication > 0) { + // check if we can delete delNodeHint + final DatanodeInfo cur; + if (firstOne && delNodeHint !=null && nonExcess.contains(delNodeHint) + && (priSet.contains(delNodeHint) + || (addedNode != null && !priSet.contains(addedNode))) ) { + cur = delNodeHint; + } else { // regular excessive replica removal + cur = replicator.chooseReplicaToDelete(inode, b, replication, + priSet, remains); + } + firstOne = false; + + // adjust rackmap, priSet, and remains + String rack = cur.getNetworkLocation(); + final List datanodes = rackMap.get(rack); + datanodes.remove(cur); + if (datanodes.isEmpty()) { + rackMap.remove(rack); + } + if (priSet.remove(cur)) { + if (datanodes.size() == 1) { + priSet.remove(datanodes.get(0)); + remains.add(datanodes.get(0)); + } + } else { + remains.remove(cur); + } + + nonExcess.remove(cur); + addToExcessReplicate(cur, b); + + // + // The 'excessblocks' tracks blocks until we get confirmation + // that the datanode has deleted them; the only way we remove them + // is when we get a "removeBlock" message. + // + // The 'invalidate' list is used to inform the datanode the block + // should be deleted. Items are removed from the invalidate list + // upon giving instructions to the namenode. + // + addToInvalidates(b, cur); + NameNode.stateChangeLog.info("BLOCK* chooseExcessReplicates: " + +"("+cur.getName()+", "+b+") is added to recentInvalidateSets"); + } + } + + private void addToExcessReplicate(DatanodeInfo dn, Block block) { assert namesystem.hasWriteLock(); Collection excessBlocks = excessReplicateMap.get(dn.getStorageID()); if (excessBlocks == null) { @@ -1834,7 +2035,7 @@ public class BlockManager { if (excessBlocks.add(block)) { excessBlocksCount++; if(NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug("BLOCK* NameSystem.chooseExcessReplicates:" + NameNode.stateChangeLog.debug("BLOCK* addToExcessReplicate:" + " (" + dn.getName() + ", " + block + ") is added to excessReplicateMap"); } @@ -1847,14 +2048,14 @@ public class BlockManager { */ private void removeStoredBlock(Block block, DatanodeDescriptor node) { if(NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug("BLOCK* NameSystem.removeStoredBlock: " + NameNode.stateChangeLog.debug("BLOCK* removeStoredBlock: " + block + " from " + node.getName()); } assert (namesystem.hasWriteLock()); { if (!blocksMap.removeNode(block, node)) { if(NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug("BLOCK* NameSystem.removeStoredBlock: " + NameNode.stateChangeLog.debug("BLOCK* removeStoredBlock: " + block + " has already been removed from node " + node); } return; @@ -1882,8 +2083,7 @@ public class BlockManager { if (excessBlocks.remove(block)) { excessBlocksCount--; if(NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug( - "BLOCK* NameSystem.removeStoredBlock: " + NameNode.stateChangeLog.debug("BLOCK* removeStoredBlock: " + block + " is removed from excessBlocks"); } if (excessBlocks.size() == 0) { @@ -1915,7 +2115,7 @@ public class BlockManager { /** * The given node is reporting that it received a certain block. */ - public void addBlock(DatanodeDescriptor node, Block block, String delHint) + private void addBlock(DatanodeDescriptor node, Block block, String delHint) throws IOException { // decrement number of blocks scheduled to this datanode. node.decBlocksScheduled(); @@ -1925,9 +2125,8 @@ public class BlockManager { if (delHint != null && delHint.length() != 0) { delHintNode = datanodeManager.getDatanode(delHint); if (delHintNode == null) { - NameNode.stateChangeLog.warn("BLOCK* NameSystem.blockReceived: " - + block + " is expected to be removed from an unrecorded node " - + delHint); + NameNode.stateChangeLog.warn("BLOCK* blockReceived: " + block + + " is expected to be removed from an unrecorded node " + delHint); } } @@ -1955,7 +2154,7 @@ public class BlockManager { addStoredBlock(b, node, delHintNode, true); } for (Block b : toInvalidate) { - NameNode.stateChangeLog.info("BLOCK* NameSystem.addBlock: block " + NameNode.stateChangeLog.info("BLOCK* addBlock: block " + b + " on " + node.getName() + " size " + b.getNumBytes() + " does not belong to any file."); addToInvalidates(b, node); @@ -1965,6 +2164,30 @@ public class BlockManager { } } + /** The given node is reporting that it received a certain block. */ + public void blockReceived(final DatanodeID nodeID, final String poolId, + final Block block, final String delHint) throws IOException { + namesystem.writeLock(); + try { + final DatanodeDescriptor node = datanodeManager.getDatanode(nodeID); + if (node == null || !node.isAlive) { + final String s = block + " is received from dead or unregistered node " + + nodeID.getName(); + NameNode.stateChangeLog.warn("BLOCK* blockReceived: " + s); + throw new IOException(s); + } + + if (NameNode.stateChangeLog.isDebugEnabled()) { + NameNode.stateChangeLog.debug("BLOCK* blockReceived: " + block + + " is received from " + nodeID.getName()); + } + + addBlock(node, block, delHint); + } finally { + namesystem.writeUnlock(); + } + } + /** * Return the number of nodes that are live and decommissioned. */ @@ -2047,7 +2270,7 @@ public class BlockManager { * On stopping decommission, check if the node has excess replicas. * If there are any excess replicas, call processOverReplicatedBlock() */ - private void processOverReplicatedBlocksOnReCommission( + void processOverReplicatedBlocksOnReCommission( final DatanodeDescriptor srcNode) { final Iterator it = srcNode.getBlockIterator(); while(it.hasNext()) { @@ -2145,9 +2368,19 @@ public class BlockManager { return blocksMap.getStoredBlock(block); } - /* updates a block in under replication queue */ - public void updateNeededReplications(Block block, int curReplicasDelta, - int expectedReplicasDelta) { + + /** Should the access keys be updated? */ + boolean shouldUpdateBlockKey(final long updateTime) throws IOException { + final boolean b = isBlockTokenEnabled && blockKeyUpdateInterval < updateTime; + if (b) { + blockTokenSecretManager.updateKeys(); + } + return b; + } + + /** updates a block in under replication queue */ + private void updateNeededReplications(final Block block, + final int curReplicasDelta, int expectedReplicasDelta) { namesystem.writeLock(); try { NumberReplicas repl = countNodes(block); @@ -2306,8 +2539,9 @@ public class BlockManager { return blocksMap.getINode(b); } - public void removeFromCorruptReplicasMap(Block block) { - corruptReplicas.removeFromCorruptReplicasMap(block); + /** @return an iterator of the datanodes. */ + public Iterator datanodeIterator(final Block block) { + return blocksMap.nodeIterator(block); } public int numCorruptReplicas(Block block) { @@ -2316,6 +2550,8 @@ public class BlockManager { public void removeBlockFromMap(Block block) { blocksMap.removeBlock(block); + // If block is removed from blocksMap remove it from corruptReplicasMap + corruptReplicas.removeFromCorruptReplicasMap(block); } public int getCapacity() { @@ -2350,63 +2586,22 @@ public class BlockManager { /** * Return an iterator over the set of blocks for which there are no replicas. */ - public BlockIterator getCorruptReplicaBlockIterator() { - return neededReplications - .iterator(UnderReplicatedBlocks.QUEUE_WITH_CORRUPT_BLOCKS); + public Iterator getCorruptReplicaBlockIterator() { + return neededReplications.iterator( + UnderReplicatedBlocks.QUEUE_WITH_CORRUPT_BLOCKS); } - /** - * Change, if appropriate, the admin state of a datanode to - * decommission completed. Return true if decommission is complete. - */ - boolean checkDecommissionStateInternal(DatanodeDescriptor node) { - // Check to see if all blocks in this decommissioned - // node has reached their target replication factor. - if (node.isDecommissionInProgress()) { - if (!isReplicationInProgress(node)) { - node.setDecommissioned(); - LOG.info("Decommission complete for node " + node.getName()); - } - } - return node.isDecommissioned(); - } - - /** Start decommissioning the specified datanode. */ - void startDecommission(DatanodeDescriptor node) throws IOException { - if (!node.isDecommissionInProgress() && !node.isDecommissioned()) { - LOG.info("Start Decommissioning node " + node.getName() + " with " + - node.numBlocks() + " blocks."); - synchronized (namesystem.heartbeats) { - namesystem.updateStats(node, false); - node.startDecommission(); - namesystem.updateStats(node, true); - } - node.decommissioningStatus.setStartTime(now()); - - // all the blocks that reside on this node have to be replicated. - checkDecommissionStateInternal(node); - } - } - - /** Stop decommissioning the specified datanodes. */ - void stopDecommission(DatanodeDescriptor node) throws IOException { - if (node.isDecommissionInProgress() || node.isDecommissioned()) { - LOG.info("Stop Decommissioning node " + node.getName()); - synchronized (namesystem.heartbeats) { - namesystem.updateStats(node, false); - node.stopDecommission(); - namesystem.updateStats(node, true); - } - processOverReplicatedBlocksOnReCommission(node); - } + /** @return the size of UnderReplicatedBlocks */ + public int numOfUnderReplicatedBlocks() { + return neededReplications.size(); } /** * Periodically calls computeReplicationWork(). */ private class ReplicationMonitor implements Runnable { - static final int INVALIDATE_WORK_PCT_PER_ITERATION = 32; - static final float REPLICATION_WORK_MULTIPLIER_PER_ITERATION = 2; + private static final int INVALIDATE_WORK_PCT_PER_ITERATION = 32; + private static final int REPLICATION_WORK_MULTIPLIER_PER_ITERATION = 2; @Override public void run() { @@ -2439,8 +2634,6 @@ public class BlockManager { */ int computeDatanodeWork() throws IOException { int workFound = 0; - int blocksToProcess = 0; - int nodesToProcess = 0; // Blocks should not be replicated or removed if in safe mode. // It's OK to check safe mode here w/o holding lock, in the worst // case extra replications will be scheduled, and these will get @@ -2448,11 +2641,11 @@ public class BlockManager { if (namesystem.isInSafeMode()) return workFound; - synchronized (namesystem.heartbeats) { - blocksToProcess = (int) (namesystem.heartbeats.size() * ReplicationMonitor.REPLICATION_WORK_MULTIPLIER_PER_ITERATION); - nodesToProcess = (int) Math.ceil((double) namesystem.heartbeats.size() - * ReplicationMonitor.INVALIDATE_WORK_PCT_PER_ITERATION / 100); - } + final int numlive = heartbeatManager.getLiveDatanodeCount(); + final int blocksToProcess = numlive + * ReplicationMonitor.REPLICATION_WORK_MULTIPLIER_PER_ITERATION; + final int nodesToProcess = (int) Math.ceil(numlive + * ReplicationMonitor.INVALIDATE_WORK_PCT_PER_ITERATION / 100.0); workFound = this.computeReplicationWork(blocksToProcess); diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index c5f8b9c53f9..fcada7b5623 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -34,7 +34,6 @@ import org.apache.hadoop.hdfs.protocol.FSConstants; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.server.namenode.FSClusterStats; import org.apache.hadoop.hdfs.server.namenode.FSInodeInfo; -import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.net.Node; import org.apache.hadoop.net.NodeBase; diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java index a41c167b0ce..e479954d42f 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java @@ -29,7 +29,7 @@ import org.apache.hadoop.hdfs.util.LightWeightGSet; * block's metadata currently includes INode it belongs to and * the datanodes that store the block. */ -public class BlocksMap { +class BlocksMap { private static class NodeIterator implements Iterator { private BlockInfo blockInfo; private int nextIdx = 0; @@ -101,7 +101,7 @@ public class BlocksMap { /** * Add block b belonging to the specified file inode to the map. */ - public BlockInfo addINode(BlockInfo b, INodeFile iNode) { + BlockInfo addINode(BlockInfo b, INodeFile iNode) { BlockInfo info = blocks.get(b); if (info != b) { info = b; @@ -137,7 +137,7 @@ public class BlocksMap { * Searches for the block in the BlocksMap and * returns Iterator that iterates through the nodes the block belongs to. */ - public Iterator nodeIterator(Block b) { + Iterator nodeIterator(Block b) { return nodeIterator(blocks.get(b)); } @@ -182,27 +182,6 @@ public class BlocksMap { Iterable getBlocks() { return blocks; } - - /** - * Check if the block exists in map - */ - public boolean contains(Block block) { - return blocks.contains(block); - } - - /** - * Check if the replica at the given datanode exists in map - */ - boolean contains(Block block, DatanodeDescriptor datanode) { - BlockInfo info = blocks.get(block); - if (info == null) - return false; - - if (-1 == info.findDatanode(datanode)) - return false; - - return true; - } /** Get the capacity of the HashMap that stores blocks */ int getCapacity() { diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index a1a56ea5b87..a069d761f3a 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; +import static org.apache.hadoop.hdfs.server.common.Util.now; + import java.io.IOException; import java.io.PrintWriter; import java.net.InetAddress; @@ -32,6 +34,7 @@ import java.util.TreeMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -49,6 +52,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTar import org.apache.hadoop.hdfs.server.common.Util; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.hdfs.server.protocol.BalancerBandwidthCommand; import org.apache.hadoop.hdfs.server.protocol.BlockCommand; import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand; import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock; @@ -56,13 +60,12 @@ import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand; import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.hdfs.server.protocol.DisallowedDatanodeException; -import org.apache.hadoop.hdfs.server.protocol.BalancerBandwidthCommand; +import org.apache.hadoop.hdfs.util.CyclicIteration; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.net.CachedDNSToSwitchMapping; import org.apache.hadoop.net.DNSToSwitchMapping; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.net.ScriptBasedMapping; -import org.apache.hadoop.util.CyclicIteration; import org.apache.hadoop.util.Daemon; import org.apache.hadoop.util.HostsFileReader; import org.apache.hadoop.util.ReflectionUtils; @@ -75,7 +78,10 @@ import org.apache.hadoop.util.ReflectionUtils; public class DatanodeManager { static final Log LOG = LogFactory.getLog(DatanodeManager.class); - final FSNamesystem namesystem; + private final FSNamesystem namesystem; + private final BlockManager blockManager; + + private final HeartbeatManager heartbeatManager; /** * Stores the datanode -> block map. @@ -117,9 +123,14 @@ public class DatanodeManager { /** Ask Datanode only up to this many blocks to delete. */ final int blockInvalidateLimit; - DatanodeManager(final FSNamesystem namesystem, final Configuration conf + DatanodeManager(final BlockManager blockManager, + final FSNamesystem namesystem, final Configuration conf ) throws IOException { this.namesystem = namesystem; + this.blockManager = blockManager; + + this.heartbeatManager = new HeartbeatManager(namesystem, conf); + this.hostsReader = new HostsFileReader( conf.get(DFSConfigKeys.DFS_HOSTS, ""), conf.get(DFSConfigKeys.DFS_HOSTS_EXCLUDE, "")); @@ -158,17 +169,30 @@ public class DatanodeManager { conf.getInt(DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_NODES_PER_INTERVAL_KEY, DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_NODES_PER_INTERVAL_DEFAULT))); decommissionthread.start(); + + heartbeatManager.activate(conf); } void close() { if (decommissionthread != null) decommissionthread.interrupt(); + heartbeatManager.close(); } /** @return the network topology. */ public NetworkTopology getNetworkTopology() { return networktopology; } - + + /** @return the heartbeat manager. */ + HeartbeatManager getHeartbeatManager() { + return heartbeatManager; + } + + /** @return the datanode statistics. */ + public DatanodeStatistics getDatanodeStatistics() { + return heartbeatManager; + } + /** Sort the located blocks by the distance to the target host. */ public void sortLocatedBlocks(final String targethost, final List locatedblocks) { @@ -231,9 +255,44 @@ public class DatanodeManager { } } + /** + * Remove a datanode descriptor. + * @param nodeInfo datanode descriptor. + */ + private void removeDatanode(DatanodeDescriptor nodeInfo) { + assert namesystem.hasWriteLock(); + heartbeatManager.removeDatanode(nodeInfo); + blockManager.removeBlocksAssociatedTo(nodeInfo); + networktopology.remove(nodeInfo); + + if (LOG.isDebugEnabled()) { + LOG.debug("remove datanode " + nodeInfo.getName()); + } + namesystem.checkSafeMode(); + } + + /** + * Remove a datanode + * @throws UnregisteredNodeException + */ + public void removeDatanode(final DatanodeID node + ) throws UnregisteredNodeException { + namesystem.writeLock(); + try { + final DatanodeDescriptor descriptor = getDatanode(node); + if (descriptor != null) { + removeDatanode(descriptor); + } else { + NameNode.stateChangeLog.warn("BLOCK* removeDatanode: " + + node.getName() + " does not exist"); + } + } finally { + namesystem.writeUnlock(); + } + } + /** Remove a dead datanode. */ - public void removeDeadDatanode(final DatanodeID nodeID) { - synchronized(namesystem.heartbeats) { + void removeDeadDatanode(final DatanodeID nodeID) { synchronized(datanodeMap) { DatanodeDescriptor d; try { @@ -244,14 +303,13 @@ public class DatanodeManager { if (d != null && isDatanodeDead(d)) { NameNode.stateChangeLog.info( "BLOCK* removeDeadDatanode: lost heartbeat from " + d.getName()); - namesystem.removeDatanode(d); + removeDatanode(d); } } - } } /** Is the datanode dead? */ - public boolean isDatanodeDead(DatanodeDescriptor node) { + boolean isDatanodeDead(DatanodeDescriptor node) { return (node.getLastUpdate() < (Util.now() - heartbeatExpireInterval)); } @@ -348,7 +406,7 @@ public class DatanodeManager { * @param nodeList * , array list of live or dead nodes. */ - public void removeDecomNodeFromList(final List nodeList) { + private void removeDecomNodeFromList(final List nodeList) { // If the include list is empty, any nodes are welcomed and it does not // make sense to exclude any nodes from the cluster. Therefore, no remove. if (hostsReader.getHosts().isEmpty()) { @@ -423,11 +481,48 @@ public class DatanodeManager { throws IOException { // If the registered node is in exclude list, then decommission it if (inExcludedHostsList(nodeReg, ipAddr)) { - namesystem.getBlockManager().startDecommission(nodeReg); + startDecommission(nodeReg); + } + } + + /** + * Change, if appropriate, the admin state of a datanode to + * decommission completed. Return true if decommission is complete. + */ + boolean checkDecommissionState(DatanodeDescriptor node) { + // Check to see if all blocks in this decommissioned + // node has reached their target replication factor. + if (node.isDecommissionInProgress()) { + if (!blockManager.isReplicationInProgress(node)) { + node.setDecommissioned(); + LOG.info("Decommission complete for node " + node.getName()); + } + } + return node.isDecommissioned(); + } + + /** Start decommissioning the specified datanode. */ + private void startDecommission(DatanodeDescriptor node) throws IOException { + if (!node.isDecommissionInProgress() && !node.isDecommissioned()) { + LOG.info("Start Decommissioning node " + node.getName() + " with " + + node.numBlocks() + " blocks."); + heartbeatManager.startDecommission(node); + node.decommissioningStatus.setStartTime(now()); + + // all the blocks that reside on this node have to be replicated. + checkDecommissionState(node); + } + } + + /** Stop decommissioning the specified datanodes. */ + void stopDecommission(DatanodeDescriptor node) throws IOException { + if (node.isDecommissionInProgress() || node.isDecommissioned()) { + LOG.info("Stop Decommissioning node " + node.getName()); + heartbeatManager.stopDecommission(node); + blockManager.processOverReplicatedBlocksOnReCommission(node); } } - /** * Generate new storage ID. * @@ -469,7 +564,7 @@ public class DatanodeManager { nodeReg.getInfoPort(), nodeReg.getIpcPort()); nodeReg.updateRegInfo(dnReg); - nodeReg.exportedKeys = namesystem.getBlockManager().getBlockKeys(); + nodeReg.exportedKeys = blockManager.getBlockKeys(); NameNode.stateChangeLog.info("BLOCK* NameSystem.registerDatanode: " + "node registration from " + nodeReg.getName() @@ -483,7 +578,7 @@ public class DatanodeManager { + "node from name: " + nodeN.getName()); // nodeN previously served a different data storage, // which is not served by anybody anymore. - namesystem.removeDatanode(nodeN); + removeDatanode(nodeN); // physically remove node from datanodeMap wipeDatanode(nodeN); nodeN = null; @@ -525,14 +620,7 @@ public class DatanodeManager { getNetworkTopology().add(nodeS); // also treat the registration message as a heartbeat - synchronized(namesystem.heartbeats) { - if( !namesystem.heartbeats.contains(nodeS)) { - namesystem.heartbeats.add(nodeS); - //update its timestamp - nodeS.updateHeartbeat(0L, 0L, 0L, 0L, 0, 0); - nodeS.isAlive = true; - } - } + heartbeatManager.register(nodeS); checkDecommissioning(nodeS, dnAddress); return; } @@ -556,16 +644,29 @@ public class DatanodeManager { checkDecommissioning(nodeDescr, dnAddress); // also treat the registration message as a heartbeat - synchronized(namesystem.heartbeats) { - namesystem.heartbeats.add(nodeDescr); - nodeDescr.isAlive = true; - // no need to update its timestamp - // because its is done when the descriptor is created + // no need to update its timestamp + // because its is done when the descriptor is created + heartbeatManager.addDatanode(nodeDescr); + } + + /** + * Rereads conf to get hosts and exclude list file names. + * Rereads the files to update the hosts and exclude lists. It + * checks if any of the hosts have changed states: + */ + public void refreshNodes(final Configuration conf) throws IOException { + namesystem.checkSuperuserPrivilege(); + refreshHostsReader(conf); + namesystem.writeLock(); + try { + refreshDatanodes(); + } finally { + namesystem.writeUnlock(); } } /** Reread include/exclude files. */ - public void refreshHostsReader(Configuration conf) throws IOException { + private void refreshHostsReader(Configuration conf) throws IOException { // Reread the conf to get dfs.hosts and dfs.hosts.exclude filenames. // Update the file names and refresh internal includes and excludes list. if (conf == null) { @@ -577,24 +678,21 @@ public class DatanodeManager { } /** - * Rereads the config to get hosts and exclude list file names. - * Rereads the files to update the hosts and exclude lists. It - * checks if any of the hosts have changed states: * 1. Added to hosts --> no further work needed here. * 2. Removed from hosts --> mark AdminState as decommissioned. * 3. Added to exclude --> start decommission. * 4. Removed from exclude --> stop decommission. */ - public void refreshDatanodes() throws IOException { + private void refreshDatanodes() throws IOException { for(DatanodeDescriptor node : datanodeMap.values()) { // Check if not include. if (!inHostsList(node, null)) { - node.setDisallowed(true); // case 2. + node.setDisallowed(true); // case 2. } else { if (inExcludedHostsList(node, null)) { - namesystem.getBlockManager().startDecommission(node); // case 3. + startDecommission(node); // case 3. } else { - namesystem.getBlockManager().stopDecommission(node); // case 4. + stopDecommission(node); // case 4. } } } @@ -626,16 +724,59 @@ public class DatanodeManager { return numDead; } + /** @return list of datanodes where decommissioning is in progress. */ + public List getDecommissioningNodes() { + namesystem.readLock(); + try { + final List decommissioningNodes + = new ArrayList(); + final List results = getDatanodeListForReport( + DatanodeReportType.LIVE); + for(DatanodeDescriptor node : results) { + if (node.isDecommissionInProgress()) { + decommissioningNodes.add(node); + } + } + return decommissioningNodes; + } finally { + namesystem.readUnlock(); + } + } + + /** Fetch live and dead datanodes. */ - public void fetchDatanodess(final List live, - final List dead) { - final List results = - getDatanodeListForReport(DatanodeReportType.ALL); - for(DatanodeDescriptor node : results) { - if (isDatanodeDead(node)) - dead.add(node); - else - live.add(node); + public void fetchDatanodes(final List live, + final List dead, final boolean removeDecommissionNode) { + if (live == null && dead == null) { + throw new HadoopIllegalArgumentException("Both live and dead lists are null"); + } + + namesystem.readLock(); + try { + final List results = + getDatanodeListForReport(DatanodeReportType.ALL); + for(DatanodeDescriptor node : results) { + if (isDatanodeDead(node)) { + if (dead != null) { + dead.add(node); + } + } else { + if (live != null) { + live.add(node); + } + } + } + } finally { + namesystem.readUnlock(); + } + + if (removeDecommissionNode) { + if (live != null) { + removeDecomNodeFromList(live); + } + if (dead != null) { + removeDecomNodeFromList(dead); + } } } @@ -712,7 +853,7 @@ public class DatanodeManager { long capacity, long dfsUsed, long remaining, long blockPoolUsed, int xceiverCount, int maxTransfers, int failedVolumes ) throws IOException { - synchronized (namesystem.heartbeats) { + synchronized (heartbeatManager) { synchronized (datanodeMap) { DatanodeDescriptor nodeinfo = null; try { @@ -731,10 +872,8 @@ public class DatanodeManager { return new DatanodeCommand[]{DatanodeCommand.REGISTER}; } - namesystem.updateStats(nodeinfo, false); - nodeinfo.updateHeartbeat(capacity, dfsUsed, remaining, blockPoolUsed, - xceiverCount, failedVolumes); - namesystem.updateStats(nodeinfo, true); + heartbeatManager.updateHeartbeat(nodeinfo, capacity, dfsUsed, + remaining, blockPoolUsed, xceiverCount, failedVolumes); //check lease recovery BlockInfoUnderConstruction[] blocks = nodeinfo @@ -765,7 +904,7 @@ public class DatanodeManager { blockPoolId, blks)); } - namesystem.addKeyUpdateCommand(cmds, nodeinfo); + blockManager.addKeyUpdateCommand(cmds, nodeinfo); // check for balancer bandwidth update if (nodeinfo.getBalancerBandwidth() > 0) { diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStatistics.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStatistics.java new file mode 100644 index 00000000000..ae8df2f50e5 --- /dev/null +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStatistics.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.blockmanagement; + +import org.apache.hadoop.hdfs.protocol.ClientProtocol; + +/** Datanode statistics */ +public interface DatanodeStatistics { + + /** @return the total capacity */ + public long getCapacityTotal(); + + /** @return the used capacity */ + public long getCapacityUsed(); + + /** @return the percentage of the used capacity over the total capacity. */ + public float getCapacityUsedPercent(); + + /** @return the remaining capacity */ + public long getCapacityRemaining(); + + /** @return the percentage of the remaining capacity over the total capacity. */ + public float getCapacityRemainingPercent(); + + /** @return the block pool used. */ + public long getBlockPoolUsed(); + + /** @return the percentage of the block pool used space over the total capacity. */ + public float getPercentBlockPoolUsed(); + + /** @return the xceiver count */ + public int getXceiverCount(); + + /** + * @return the total used space by data nodes for non-DFS purposes + * such as storing temporary files on the local file system + */ + public long getCapacityUsedNonDFS(); + + /** The same as {@link ClientProtocol#getStats()}. + * The block related entries are set to -1. + */ + public long[] getStats(); +} \ No newline at end of file diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java index 8275decec36..93afe65118a 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java @@ -24,7 +24,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; -import org.apache.hadoop.util.CyclicIteration; /** * Manage node decommissioning. @@ -35,11 +34,9 @@ class DecommissionManager { static final Log LOG = LogFactory.getLog(DecommissionManager.class); private final FSNamesystem fsnamesystem; - private final BlockManager blockManager; - DecommissionManager(FSNamesystem namesystem) { + DecommissionManager(final FSNamesystem namesystem) { this.fsnamesystem = namesystem; - this.blockManager = fsnamesystem.getBlockManager(); } /** Periodically check decommission status. */ @@ -81,16 +78,16 @@ class DecommissionManager { } private void check() { + final DatanodeManager dm = fsnamesystem.getBlockManager().getDatanodeManager(); int count = 0; for(Map.Entry entry - : blockManager.getDatanodeManager().getDatanodeCyclicIteration( - firstkey)) { + : dm.getDatanodeCyclicIteration(firstkey)) { final DatanodeDescriptor d = entry.getValue(); firstkey = entry.getKey(); if (d.isDecommissionInProgress()) { try { - blockManager.checkDecommissionStateInternal(d); + dm.checkDecommissionState(d); } catch(Exception e) { LOG.warn("entry=" + entry, e); } diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java new file mode 100644 index 00000000000..266850c02db --- /dev/null +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java @@ -0,0 +1,301 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.blockmanagement; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdfs.protocol.DatanodeID; +import org.apache.hadoop.hdfs.server.common.Util; +import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.util.Daemon; + +/** + * Manage the heartbeats received from datanodes. + * The datanode list and statistics are synchronized + * by the heartbeat manager lock. + */ +class HeartbeatManager implements DatanodeStatistics { + static final Log LOG = LogFactory.getLog(HeartbeatManager.class); + + /** + * Stores a subset of the datanodeMap in DatanodeManager, + * containing nodes that are considered alive. + * The HeartbeatMonitor periodically checks for out-dated entries, + * and removes them from the list. + * It is synchronized by the heartbeat manager lock. + */ + private final List datanodes = new ArrayList(); + + /** Statistics, which are synchronized by the heartbeat manager lock. */ + private final Stats stats = new Stats(); + + /** The time period to check for expired datanodes */ + private final long heartbeatRecheckInterval; + /** Heartbeat monitor thread */ + private final Daemon heartbeatThread = new Daemon(new Monitor()); + + final FSNamesystem namesystem; + + HeartbeatManager(final FSNamesystem namesystem, final Configuration conf) { + this.heartbeatRecheckInterval = conf.getInt( + DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, + DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_DEFAULT); // 5 minutes + + this.namesystem = namesystem; + } + + void activate(Configuration conf) { + heartbeatThread.start(); + } + + void close() { + heartbeatThread.interrupt(); + } + + synchronized int getLiveDatanodeCount() { + return datanodes.size(); + } + + @Override + public synchronized long getCapacityTotal() { + return stats.capacityTotal; + } + + @Override + public synchronized long getCapacityUsed() { + return stats.capacityUsed; + } + + @Override + public synchronized float getCapacityUsedPercent() { + return DFSUtil.getPercentUsed(stats.capacityUsed, stats.capacityTotal); + } + + @Override + public synchronized long getCapacityRemaining() { + return stats.capacityRemaining; + } + + @Override + public synchronized float getCapacityRemainingPercent() { + return DFSUtil.getPercentRemaining( + stats.capacityRemaining, stats.capacityTotal); + } + + @Override + public synchronized long getBlockPoolUsed() { + return stats.blockPoolUsed; + } + + @Override + public synchronized float getPercentBlockPoolUsed() { + return DFSUtil.getPercentUsed(stats.blockPoolUsed, stats.capacityTotal); + } + + @Override + public synchronized long getCapacityUsedNonDFS() { + final long nonDFSUsed = stats.capacityTotal + - stats.capacityRemaining - stats.capacityUsed; + return nonDFSUsed < 0L? 0L : nonDFSUsed; + } + + @Override + public synchronized int getXceiverCount() { + return stats.xceiverCount; + } + + @Override + public synchronized long[] getStats() { + return new long[] {getCapacityTotal(), + getCapacityUsed(), + getCapacityRemaining(), + -1L, + -1L, + -1L, + getBlockPoolUsed()}; + } + + synchronized void register(final DatanodeDescriptor d) { + if (!datanodes.contains(d)) { + addDatanode(d); + + //update its timestamp + d.updateHeartbeat(0L, 0L, 0L, 0L, 0, 0); + } + } + + synchronized DatanodeDescriptor[] getDatanodes() { + return datanodes.toArray(new DatanodeDescriptor[datanodes.size()]); + } + + synchronized void addDatanode(final DatanodeDescriptor d) { + datanodes.add(d); + d.isAlive = true; + } + + synchronized void removeDatanode(DatanodeDescriptor node) { + if (node.isAlive) { + stats.subtract(node); + datanodes.remove(node); + node.isAlive = false; + } + } + + synchronized void updateHeartbeat(final DatanodeDescriptor node, + long capacity, long dfsUsed, long remaining, long blockPoolUsed, + int xceiverCount, int failedVolumes) { + stats.subtract(node); + node.updateHeartbeat(capacity, dfsUsed, remaining, blockPoolUsed, + xceiverCount, failedVolumes); + stats.add(node); + } + + synchronized void startDecommission(final DatanodeDescriptor node) { + stats.subtract(node); + node.startDecommission(); + stats.add(node); + } + + synchronized void stopDecommission(final DatanodeDescriptor node) { + stats.subtract(node); + node.stopDecommission(); + stats.add(node); + } + + /** + * Check if there are any expired heartbeats, and if so, + * whether any blocks have to be re-replicated. + * While removing dead datanodes, make sure that only one datanode is marked + * dead at a time within the synchronized section. Otherwise, a cascading + * effect causes more datanodes to be declared dead. + */ + void heartbeatCheck() { + final DatanodeManager dm = namesystem.getBlockManager().getDatanodeManager(); + // It's OK to check safe mode w/o taking the lock here, we re-check + // for safe mode after taking the lock before removing a datanode. + if (namesystem.isInSafeMode()) { + return; + } + boolean allAlive = false; + while (!allAlive) { + // locate the first dead node. + DatanodeID dead = null; + synchronized(this) { + for (DatanodeDescriptor d : datanodes) { + if (dm.isDatanodeDead(d)) { + namesystem.incrExpiredHeartbeats(); + dead = d; + break; + } + } + } + + allAlive = dead == null; + if (!allAlive) { + // acquire the fsnamesystem lock, and then remove the dead node. + namesystem.writeLock(); + if (namesystem.isInSafeMode()) { + return; + } + try { + synchronized(this) { + dm.removeDeadDatanode(dead); + } + } finally { + namesystem.writeUnlock(); + } + } + } + } + + + /** Periodically check heartbeat and update block key */ + private class Monitor implements Runnable { + private long lastHeartbeatCheck; + private long lastBlockKeyUpdate; + + @Override + public void run() { + while(namesystem.isRunning()) { + try { + final long now = Util.now(); + if (lastHeartbeatCheck + heartbeatRecheckInterval < now) { + heartbeatCheck(); + lastHeartbeatCheck = now; + } + if (namesystem.getBlockManager().shouldUpdateBlockKey( + now - lastBlockKeyUpdate)) { + synchronized(HeartbeatManager.this) { + for(DatanodeDescriptor d : datanodes) { + d.needKeyUpdate = true; + } + } + lastBlockKeyUpdate = now; + } + } catch (Exception e) { + LOG.error("Exception while checking heartbeat", e); + } + try { + Thread.sleep(5000); // 5 seconds + } catch (InterruptedException ie) { + } + } + } + } + + /** Datanode statistics. + * For decommissioning/decommissioned nodes, only used capacity is counted. + */ + private static class Stats { + private long capacityTotal = 0L; + private long capacityUsed = 0L; + private long capacityRemaining = 0L; + private long blockPoolUsed = 0L; + private int xceiverCount = 0; + + private void add(final DatanodeDescriptor node) { + capacityUsed += node.getDfsUsed(); + blockPoolUsed += node.getBlockPoolUsed(); + xceiverCount += node.getXceiverCount(); + if (!(node.isDecommissionInProgress() || node.isDecommissioned())) { + capacityTotal += node.getCapacity(); + capacityRemaining += node.getRemaining(); + } else { + capacityTotal += node.getDfsUsed(); + } + } + + private void subtract(final DatanodeDescriptor node) { + capacityUsed -= node.getDfsUsed(); + blockPoolUsed -= node.getBlockPoolUsed(); + xceiverCount -= node.getXceiverCount(); + if (!(node.isDecommissionInProgress() || node.isDecommissioned())) { + capacityTotal -= node.getCapacity(); + capacityRemaining -= node.getRemaining(); + } else { + capacityTotal -= node.getDfsUsed(); + } + } + } +} diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java index 9c2a213985c..7b39860d06f 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java @@ -17,21 +17,26 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import java.util.*; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.NavigableSet; +import java.util.TreeSet; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.namenode.NameNode; -/* Class for keeping track of under replication blocks +/** Keep track of under replication blocks. * Blocks have replication priority, with priority 0 indicating the highest * Blocks have only one replicas has the highest */ -public class UnderReplicatedBlocks implements Iterable { +class UnderReplicatedBlocks implements Iterable { static final int LEVEL = 5; - static public final int QUEUE_WITH_CORRUPT_BLOCKS = 4; - private List> priorityQueues = new ArrayList>(); + static final int QUEUE_WITH_CORRUPT_BLOCKS = 4; + private final List> priorityQueues + = new ArrayList>(); - /* constructor */ + /** Create an object. */ UnderReplicatedBlocks() { for(int i=0; i()); @@ -47,8 +52,8 @@ public class UnderReplicatedBlocks implements Iterable { } } - /* Return the total number of under replication blocks */ - public synchronized int size() { + /** Return the total number of under replication blocks */ + synchronized int size() { int size = 0; for (int i=0; i { return size; } - /* Return the number of under replication blocks excluding corrupt blocks */ + /** Return the number of under replication blocks excluding corrupt blocks */ synchronized int getUnderReplicatedBlockCount() { int size = 0; for (int i=0; i { return priorityQueues.get(QUEUE_WITH_CORRUPT_BLOCKS).size(); } - /* Check if a block is in the neededReplication queue */ - public synchronized boolean contains(Block block) { - for(TreeSet set:priorityQueues) { + /** Check if a block is in the neededReplication queue */ + synchronized boolean contains(Block block) { + for(NavigableSet set : priorityQueues) { if(set.contains(block)) { return true; } } return false; } - /* Return the priority of a block + /** Return the priority of a block * @param block a under replication block * @param curReplicas current number of replicas of the block * @param expectedReplicas expected number of replicas of the block @@ -106,7 +111,7 @@ public class UnderReplicatedBlocks implements Iterable { } } - /* add a block to a under replication queue according to its priority + /** add a block to a under replication queue according to its priority * @param block a under replication block * @param curReplicas current number of replicas of the block * @param expectedReplicas expected number of replicas of the block @@ -134,7 +139,7 @@ public class UnderReplicatedBlocks implements Iterable { return false; } - /* remove a block from a under replication queue */ + /** remove a block from a under replication queue */ synchronized boolean remove(Block block, int oldReplicas, int decommissionedReplicas, @@ -145,7 +150,7 @@ public class UnderReplicatedBlocks implements Iterable { return remove(block, priLevel); } - /* remove a block from a under replication queue given a priority*/ + /** remove a block from a under replication queue given a priority*/ boolean remove(Block block, int priLevel) { if(priLevel >= 0 && priLevel < LEVEL && priorityQueues.get(priLevel).remove(block)) { @@ -174,7 +179,7 @@ public class UnderReplicatedBlocks implements Iterable { return false; } - /* update the priority level of a block */ + /** update the priority level of a block */ synchronized void update(Block block, int curReplicas, int decommissionedReplicas, int curExpectedReplicas, @@ -209,30 +214,29 @@ public class UnderReplicatedBlocks implements Iterable { } } - /* returns an iterator of all blocks in a given priority queue */ + /** returns an iterator of all blocks in a given priority queue */ synchronized BlockIterator iterator(int level) { return new BlockIterator(level); } - /* return an iterator of all the under replication blocks */ + /** return an iterator of all the under replication blocks */ public synchronized BlockIterator iterator() { return new BlockIterator(); } - public class BlockIterator implements Iterator { + class BlockIterator implements Iterator { private int level; private boolean isIteratorForLevel = false; private List> iterators = new ArrayList>(); - BlockIterator() - { + private BlockIterator() { level=0; for(int i=0; i { } } + @Override public Block next() { if (isIteratorForLevel) return iterators.get(0).next(); @@ -253,6 +258,7 @@ public class UnderReplicatedBlocks implements Iterable { return iterators.get(level).next(); } + @Override public boolean hasNext() { if (isIteratorForLevel) return iterators.get(0).hasNext(); @@ -260,6 +266,7 @@ public class UnderReplicatedBlocks implements Iterable { return iterators.get(level).hasNext(); } + @Override public void remove() { if (isIteratorForLevel) iterators.get(0).remove(); @@ -267,8 +274,8 @@ public class UnderReplicatedBlocks implements Iterable { iterators.get(level).remove(); } - public int getPriority() { + int getPriority() { return level; - }; + } } } diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/common/HdfsConstants.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/common/HdfsConstants.java index 755ecc69355..ef83ad2db83 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/common/HdfsConstants.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/common/HdfsConstants.java @@ -29,7 +29,10 @@ import org.apache.hadoop.classification.InterfaceAudience; ************************************/ @InterfaceAudience.Private -public interface HdfsConstants { +public final class HdfsConstants { + /* Hidden constructor */ + private HdfsConstants() { } + /** * Type of the node */ diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/common/JspHelper.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/common/JspHelper.java index a2b7ca69973..cf2452992f5 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/common/JspHelper.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/common/JspHelper.java @@ -26,11 +26,11 @@ import java.net.InetSocketAddress; import java.net.Socket; import java.net.URL; import java.net.URLEncoder; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.List; import java.util.TreeSet; import javax.servlet.ServletContext; @@ -190,13 +190,15 @@ public class JspHelper { s.connect(addr, HdfsConstants.READ_TIMEOUT); s.setSoTimeout(HdfsConstants.READ_TIMEOUT); - long amtToRead = Math.min(chunkSizeToView, blockSize - offsetIntoBlock); + long amtToRead = Math.min(chunkSizeToView, blockSize - offsetIntoBlock); // Use the block name for file name. - String file = BlockReader.getFileName(addr, poolId, blockId); - BlockReader blockReader = BlockReader.newBlockReader(s, file, + int bufferSize = conf.getInt(DFSConfigKeys.IO_FILE_BUFFER_SIZE_KEY, + DFSConfigKeys.IO_FILE_BUFFER_SIZE_DEFAULT); + String file = BlockReader.getFileName(addr, poolId, blockId); + BlockReader blockReader = BlockReader.newBlockReader(s, file, new ExtendedBlock(poolId, blockId, 0, genStamp), blockToken, - offsetIntoBlock, amtToRead, conf.getInt("io.file.buffer.size", 4096)); + offsetIntoBlock, amtToRead, bufferSize); byte[] buf = new byte[(int)amtToRead]; int readOffset = 0; @@ -249,7 +251,7 @@ public class JspHelper { out.print(""); } - public static void sortNodeList(ArrayList nodes, + public static void sortNodeList(final List nodes, String field, String order) { class NodeComapare implements Comparator { diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java index 1ea9dfbcae6..b51241ed3fe 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java @@ -54,7 +54,7 @@ import org.apache.hadoop.util.PureJavaCrc32; * may copies it to another site. If a throttler is provided, * streaming throttling is also supported. **/ -class BlockReceiver implements Closeable, FSConstants { +class BlockReceiver implements Closeable { public static final Log LOG = DataNode.LOG; static final Log ClientTraceLog = DataNode.ClientTraceLog; @@ -179,8 +179,7 @@ class BlockReceiver implements Closeable, FSConstants { this.out = streams.dataOut; this.cout = streams.checksumOut; this.checksumOut = new DataOutputStream(new BufferedOutputStream( - streams.checksumOut, - SMALL_BUFFER_SIZE)); + streams.checksumOut, FSConstants.SMALL_BUFFER_SIZE)); // write data chunk header if creating a new replica if (isCreate) { BlockMetadataHeader.writeHeader(checksumOut, checksum); @@ -399,7 +398,7 @@ class BlockReceiver implements Closeable, FSConstants { buf.limit(bufRead); } - while (buf.remaining() < SIZE_OF_INTEGER) { + while (buf.remaining() < FSConstants.BYTES_IN_INTEGER) { if (buf.position() > 0) { shiftBufData(); } @@ -418,9 +417,10 @@ class BlockReceiver implements Closeable, FSConstants { payloadLen); } - // Subtract SIZE_OF_INTEGER since that accounts for the payloadLen that + // Subtract BYTES_IN_INTEGER since that accounts for the payloadLen that // we read above. - int pktSize = payloadLen + PacketHeader.PKT_HEADER_LEN - SIZE_OF_INTEGER; + int pktSize = payloadLen + PacketHeader.PKT_HEADER_LEN + - FSConstants.BYTES_IN_INTEGER; if (buf.remaining() < pktSize) { //we need to read more data @@ -817,7 +817,7 @@ class BlockReceiver implements Closeable, FSConstants { * Processed responses from downstream datanodes in the pipeline * and sends back replies to the originator. */ - class PacketResponder implements Runnable, Closeable, FSConstants { + class PacketResponder implements Runnable, Closeable { /** queue for packets waiting for ack */ private final LinkedList ackQueue = new LinkedList(); diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java index b78f00ef0b9..ac194622a32 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java @@ -42,7 +42,7 @@ import org.apache.hadoop.util.DataChecksum; /** * Reads a block from the disk and sends it to a recipient. */ -class BlockSender implements java.io.Closeable, FSConstants { +class BlockSender implements java.io.Closeable { public static final Log LOG = DataNode.LOG; static final Log ClientTraceLog = DataNode.ClientTraceLog; @@ -155,7 +155,7 @@ class BlockSender implements java.io.Closeable, FSConstants { if ( !corruptChecksumOk || datanode.data.metaFileExists(block) ) { checksumIn = new DataInputStream(new BufferedInputStream(datanode.data - .getMetaDataInputStream(block), BUFFER_SIZE)); + .getMetaDataInputStream(block), FSConstants.IO_FILE_BUFFER_SIZE)); // read and handle the common header here. For now just a version BlockMetadataHeader header = BlockMetadataHeader.readHeader(checksumIn); @@ -472,15 +472,15 @@ class BlockSender implements java.io.Closeable, FSConstants { streamForSendChunks = baseStream; // assure a mininum buffer size. - maxChunksPerPacket = (Math.max(BUFFER_SIZE, + maxChunksPerPacket = (Math.max(FSConstants.IO_FILE_BUFFER_SIZE, MIN_BUFFER_WITH_TRANSFERTO) + bytesPerChecksum - 1)/bytesPerChecksum; // allocate smaller buffer while using transferTo(). pktSize += checksumSize * maxChunksPerPacket; } else { - maxChunksPerPacket = Math.max(1, - (BUFFER_SIZE + bytesPerChecksum - 1)/bytesPerChecksum); + maxChunksPerPacket = Math.max(1, (FSConstants.IO_FILE_BUFFER_SIZE + + bytesPerChecksum - 1) / bytesPerChecksum); pktSize += (bytesPerChecksum + checksumSize) * maxChunksPerPacket; } diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index c545b29fb05..f94dc9b72c2 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -169,7 +169,7 @@ import org.mortbay.util.ajax.JSON; **********************************************************/ @InterfaceAudience.Private public class DataNode extends Configured - implements InterDatanodeProtocol, ClientDatanodeProtocol, FSConstants, + implements InterDatanodeProtocol, ClientDatanodeProtocol, DataNodeMXBean { public static final Log LOG = LogFactory.getLog(DataNode.class); @@ -348,7 +348,7 @@ public class DataNode extends Configured ThreadGroup threadGroup = null; long blockReportInterval; boolean resetBlockReportTime = true; - long initialBlockReportDelay = BLOCKREPORT_INITIAL_DELAY * 1000L; + long initialBlockReportDelay = DFS_BLOCKREPORT_INTERVAL_MSEC_DEFAULT * 1000L; long heartBeatInterval; private boolean heartbeatsDisabledForTests = false; private DataStorage storage = null; @@ -440,21 +440,23 @@ public class DataNode extends Configured HdfsConstants.WRITE_TIMEOUT); /* Based on results on different platforms, we might need set the default * to false on some of them. */ - this.transferToAllowed = conf.getBoolean("dfs.datanode.transferTo.allowed", - true); + this.transferToAllowed = conf.getBoolean( + DFS_DATANODE_TRANSFERTO_ALLOWED_KEY, + DFS_DATANODE_TRANSFERTO_ALLOWED_DEFAULT); this.writePacketSize = conf.getInt(DFS_CLIENT_WRITE_PACKET_SIZE_KEY, DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT); - - this.blockReportInterval = - conf.getLong(DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, BLOCKREPORT_INTERVAL); - this.initialBlockReportDelay = conf.getLong(DFS_BLOCKREPORT_INITIAL_DELAY_KEY, - BLOCKREPORT_INITIAL_DELAY)* 1000L; + this.blockReportInterval = conf.getLong(DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, + DFS_BLOCKREPORT_INTERVAL_MSEC_DEFAULT); + this.initialBlockReportDelay = conf.getLong( + DFS_BLOCKREPORT_INITIAL_DELAY_KEY, + DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT) * 1000L; if (this.initialBlockReportDelay >= blockReportInterval) { this.initialBlockReportDelay = 0; LOG.info("dfs.blockreport.initialDelay is greater than " + "dfs.blockreport.intervalMsec." + " Setting initial delay to 0 msec:"); } - this.heartBeatInterval = conf.getLong(DFS_HEARTBEAT_INTERVAL_KEY, HEARTBEAT_INTERVAL) * 1000L; + this.heartBeatInterval = conf.getLong(DFS_HEARTBEAT_INTERVAL_KEY, + DFS_HEARTBEAT_INTERVAL_DEFAULT) * 1000L; // do we need to sync block file contents to disk when blockfile is closed? this.syncOnClose = conf.getBoolean(DFS_DATANODE_SYNCONCLOSE_KEY, @@ -617,7 +619,7 @@ public class DataNode extends Configured } else { ss = secureResources.getStreamingSocket(); } - ss.setReceiveBufferSize(DEFAULT_DATA_SOCKET_SIZE); + ss.setReceiveBufferSize(FSConstants.DEFAULT_DATA_SOCKET_SIZE); // adjust machine name with the actual port int tmpPort = ss.getLocalPort(); selfAddr = new InetSocketAddress(ss.getInetAddress().getHostAddress(), @@ -1967,8 +1969,8 @@ public class DataNode extends Configured long writeTimeout = socketWriteTimeout + HdfsConstants.WRITE_TIMEOUT_EXTENSION * (targets.length-1); OutputStream baseStream = NetUtils.getOutputStream(sock, writeTimeout); - out = new DataOutputStream(new BufferedOutputStream(baseStream, - SMALL_BUFFER_SIZE)); + out = new DataOutputStream(new BufferedOutputStream(baseStream, + FSConstants.SMALL_BUFFER_SIZE)); blockSender = new BlockSender(b, 0, b.getNumBytes(), false, false, false, DataNode.this); DatanodeInfo srcNode = new DatanodeInfo(bpReg); diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java index ab99a377696..384d2b93aa1 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java @@ -69,7 +69,7 @@ import com.google.protobuf.ByteString; /** * Thread for processing incoming/outgoing data stream. */ -class DataXceiver extends Receiver implements Runnable, FSConstants { +class DataXceiver extends Receiver implements Runnable { public static final Log LOG = DataNode.LOG; static final Log ClientTraceLog = DataNode.ClientTraceLog; @@ -202,8 +202,8 @@ class DataXceiver extends Receiver implements Runnable, FSConstants { final long length) throws IOException { OutputStream baseStream = NetUtils.getOutputStream(s, datanode.socketWriteTimeout); - DataOutputStream out = new DataOutputStream( - new BufferedOutputStream(baseStream, SMALL_BUFFER_SIZE)); + DataOutputStream out = new DataOutputStream(new BufferedOutputStream( + baseStream, FSConstants.SMALL_BUFFER_SIZE)); checkAccess(out, true, block, blockToken, Op.READ_BLOCK, BlockTokenSecretManager.AccessMode.READ); @@ -329,7 +329,7 @@ class DataXceiver extends Receiver implements Runnable, FSConstants { final DataOutputStream replyOut = new DataOutputStream( new BufferedOutputStream( NetUtils.getOutputStream(s, datanode.socketWriteTimeout), - SMALL_BUFFER_SIZE)); + FSConstants.SMALL_BUFFER_SIZE)); checkAccess(replyOut, isClient, block, blockToken, Op.WRITE_BLOCK, BlockTokenSecretManager.AccessMode.WRITE); @@ -369,11 +369,11 @@ class DataXceiver extends Receiver implements Runnable, FSConstants { (HdfsConstants.WRITE_TIMEOUT_EXTENSION * targets.length); NetUtils.connect(mirrorSock, mirrorTarget, timeoutValue); mirrorSock.setSoTimeout(timeoutValue); - mirrorSock.setSendBufferSize(DEFAULT_DATA_SOCKET_SIZE); + mirrorSock.setSendBufferSize(FSConstants.DEFAULT_DATA_SOCKET_SIZE); mirrorOut = new DataOutputStream( new BufferedOutputStream( NetUtils.getOutputStream(mirrorSock, writeTimeout), - SMALL_BUFFER_SIZE)); + FSConstants.SMALL_BUFFER_SIZE)); mirrorIn = new DataInputStream(NetUtils.getInputStream(mirrorSock)); new Sender(mirrorOut).writeBlock(originalBlock, blockToken, @@ -524,7 +524,7 @@ class DataXceiver extends Receiver implements Runnable, FSConstants { final MetaDataInputStream metadataIn = datanode.data.getMetaDataInputStream(block); final DataInputStream checksumIn = new DataInputStream(new BufferedInputStream( - metadataIn, BUFFER_SIZE)); + metadataIn, FSConstants.IO_FILE_BUFFER_SIZE)); updateCurrentThreadName("Getting checksum for block " + block); try { @@ -603,7 +603,7 @@ class DataXceiver extends Receiver implements Runnable, FSConstants { OutputStream baseStream = NetUtils.getOutputStream( s, datanode.socketWriteTimeout); reply = new DataOutputStream(new BufferedOutputStream( - baseStream, SMALL_BUFFER_SIZE)); + baseStream, FSConstants.SMALL_BUFFER_SIZE)); // send status first writeResponse(SUCCESS, reply); @@ -681,15 +681,15 @@ class DataXceiver extends Receiver implements Runnable, FSConstants { OutputStream baseStream = NetUtils.getOutputStream(proxySock, datanode.socketWriteTimeout); - proxyOut = new DataOutputStream( - new BufferedOutputStream(baseStream, SMALL_BUFFER_SIZE)); + proxyOut = new DataOutputStream(new BufferedOutputStream(baseStream, + FSConstants.SMALL_BUFFER_SIZE)); /* send request to the proxy */ new Sender(proxyOut).copyBlock(block, blockToken); // receive the response from the proxy proxyReply = new DataInputStream(new BufferedInputStream( - NetUtils.getInputStream(proxySock), BUFFER_SIZE)); + NetUtils.getInputStream(proxySock), FSConstants.IO_FILE_BUFFER_SIZE)); BlockOpResponseProto copyResponse = BlockOpResponseProto.parseFrom( HdfsProtoUtil.vintPrefixed(proxyReply)); diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java index e7274c300aa..32902b7d102 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java @@ -42,7 +42,7 @@ import org.apache.hadoop.util.Daemon; * other DataNodes. This small server does not use the * Hadoop IPC mechanism. */ -class DataXceiverServer implements Runnable, FSConstants { +class DataXceiverServer implements Runnable { public static final Log LOG = DataNode.LOG; ServerSocket ss; @@ -119,8 +119,8 @@ class DataXceiverServer implements Runnable, FSConstants { conf.getInt(DFSConfigKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_KEY, DFSConfigKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_DEFAULT); - this.estimateBlockSize = - conf.getLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE); + this.estimateBlockSize = conf.getLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, + DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT); //set up parameter for cluster balancing this.balanceThrottler = new BlockBalanceThrottler( diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java index fb10ab850e6..6217bb6fc47 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java @@ -47,8 +47,8 @@ import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.util.ServletUtil; import org.apache.hadoop.util.StringUtils; -import org.mortbay.util.URIUtil; @InterfaceAudience.Private public class DatanodeJspHelper { @@ -289,7 +289,7 @@ public class DatanodeJspHelper { // Add the various links for looking at the file contents // URL for downloading the full file String downloadUrl = "http://" + req.getServerName() + ":" - + req.getServerPort() + "/streamFile" + URIUtil.encodePath(filename) + + req.getServerPort() + "/streamFile" + ServletUtil.encodePath(filename) + JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr, true) + JspHelper.getDelegationTokenUrlParam(tokenString); out.print(""); diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java index f12b8403b35..2d827ac5b72 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java @@ -75,7 +75,7 @@ import org.apache.hadoop.util.ReflectionUtils; * ***************************************************/ @InterfaceAudience.Private -public class FSDataset implements FSConstants, FSDatasetInterface { +public class FSDataset implements FSDatasetInterface { /** * A node type that can be built into a tree reflecting the @@ -465,7 +465,7 @@ public class FSDataset implements FSConstants, FSDatasetInterface { } checksumIn = new DataInputStream( new BufferedInputStream(new FileInputStream(metaFile), - BUFFER_SIZE)); + FSConstants.IO_FILE_BUFFER_SIZE)); // read and handle the common header here. For now just a version BlockMetadataHeader header = BlockMetadataHeader.readHeader(checksumIn); @@ -775,12 +775,13 @@ public class FSDataset implements FSConstants, FSDatasetInterface { */ private volatile List volumes = null; BlockVolumeChoosingPolicy blockChooser; - int numFailedVolumes = 0; + int numFailedVolumes; - FSVolumeSet(FSVolume[] volumes, BlockVolumeChoosingPolicy blockChooser) { + FSVolumeSet(FSVolume[] volumes, int failedVols, BlockVolumeChoosingPolicy blockChooser) { List list = Arrays.asList(volumes); this.volumes = Collections.unmodifiableList(list); this.blockChooser = blockChooser; + this.numFailedVolumes = failedVols; } private int numberOfVolumes() { @@ -1144,15 +1145,19 @@ public class FSDataset implements FSConstants, FSDatasetInterface { String[] dataDirs = conf.getTrimmedStrings(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY); int volsConfigured = (dataDirs == null) ? 0 : dataDirs.length; - + int volsFailed = volsConfigured - storage.getNumStorageDirs(); this.validVolsRequired = volsConfigured - volFailuresTolerated; - if (validVolsRequired < 1 - || validVolsRequired > storage.getNumStorageDirs()) { + if (volFailuresTolerated < 0 || volFailuresTolerated >= volsConfigured) { + throw new DiskErrorException("Invalid volume failure " + + " config value: " + volFailuresTolerated); + } + if (volsFailed > volFailuresTolerated) { throw new DiskErrorException("Too many failed volumes - " + "current valid volumes: " + storage.getNumStorageDirs() + ", volumes configured: " + volsConfigured - + ", volume failures tolerated: " + volFailuresTolerated ); + + ", volumes failed: " + volsFailed + + ", volume failures tolerated: " + volFailuresTolerated); } FSVolume[] volArray = new FSVolume[storage.getNumStorageDirs()]; @@ -1170,7 +1175,7 @@ public class FSDataset implements FSConstants, FSDatasetInterface { RoundRobinVolumesPolicy.class, BlockVolumeChoosingPolicy.class), conf); - volumes = new FSVolumeSet(volArray, blockChooserImpl); + volumes = new FSVolumeSet(volArray, volsFailed, blockChooserImpl); volumes.getVolumeMap(volumeMap); File[] roots = new File[storage.getNumStorageDirs()]; diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java index eca31fe4d04..d72509cee2e 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java @@ -91,7 +91,6 @@ public class BackupImage extends FSImage { super(conf); storage.setDisablePreUpgradableLayoutCheck(true); bnState = BNState.DROP_UNTIL_NEXT_ROLL; - editLog.initJournals(); } /** @@ -210,14 +209,13 @@ public class BackupImage extends FSImage { if (LOG.isTraceEnabled()) { LOG.debug("data:" + StringUtils.byteToHexString(data)); } - backupInputStream.setBytes(data); + FSEditLogLoader logLoader = new FSEditLogLoader(namesystem); int logVersion = storage.getLayoutVersion(); - BufferedInputStream bin = new BufferedInputStream(backupInputStream); - DataInputStream in = new DataInputStream(bin); - Checksum checksum = FSEditLog.getChecksum(); - int numLoaded = logLoader.loadEditRecords(logVersion, in, checksum, true, - lastAppliedTxId + 1); + backupInputStream.setBytes(data, logVersion); + + int numLoaded = logLoader.loadEditRecords(logVersion, backupInputStream, + true, lastAppliedTxId + 1); if (numLoaded != numTxns) { throw new IOException("Batch of txns starting at txnid " + firstTxId + " was supposed to contain " + numTxns + diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java index 35c4b7384f7..3cac6676f13 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java @@ -54,7 +54,7 @@ class BackupJournalManager implements JournalManager { } @Override - public void purgeLogsOlderThan(long minTxIdToKeep, StoragePurger purger) + public void purgeLogsOlderThan(long minTxIdToKeep) throws IOException { } diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java index 49f713015c4..f75410031d3 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java @@ -207,7 +207,7 @@ class Checkpointer extends Daemon { long lastApplied = bnImage.getLastAppliedTxId(); LOG.debug("Doing checkpoint. Last applied: " + lastApplied); RemoteEditLogManifest manifest = - getNamenode().getEditLogManifest(bnImage.getLastAppliedTxId()); + getNamenode().getEditLogManifest(bnImage.getLastAppliedTxId() + 1); if (!manifest.getLogs().isEmpty()) { RemoteEditLog firstRemoteLog = manifest.getLogs().get(0); diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryServlet.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryServlet.java index acf473a5a7c..6ed5f495548 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryServlet.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryServlet.java @@ -31,6 +31,7 @@ import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.ServletUtil; import org.znerd.xmlenc.XMLOutputter; /** Servlets for file checksum */ @@ -49,8 +50,7 @@ public class ContentSummaryServlet extends DfsServlet { ugi.doAs(new PrivilegedExceptionAction() { @Override public Void run() throws Exception { - final String path = request.getPathInfo(); - + final String path = ServletUtil.getDecodedPath(request, "/contentSummary"); final PrintWriter out = response.getWriter(); final XMLOutputter xml = new XMLOutputter(out, "UTF-8"); xml.declaration(); diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/DfsServlet.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/DfsServlet.java index cee3e8d3a5e..ea0f392a3d8 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/DfsServlet.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/DfsServlet.java @@ -19,8 +19,6 @@ package org.apache.hadoop.hdfs.server.namenode; import java.io.IOException; import java.net.InetSocketAddress; -import java.net.URI; -import java.net.URISyntaxException; import javax.servlet.ServletContext; import javax.servlet.http.HttpServlet; @@ -33,8 +31,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.protocol.ClientProtocol; -import org.apache.hadoop.hdfs.protocol.DatanodeID; -import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.security.UserGroupInformation; @@ -86,48 +82,6 @@ abstract class DfsServlet extends HttpServlet { return DFSUtil.createNamenode(nnAddr, conf); } - /** Create a URI for redirecting request to a datanode */ - protected URI createRedirectUri(String servletpath, - UserGroupInformation ugi, - DatanodeID host, - HttpServletRequest request, - NameNode nn - ) throws IOException, URISyntaxException { - final String hostname = host instanceof DatanodeInfo? - ((DatanodeInfo)host).getHostName(): host.getHost(); - final String scheme = request.getScheme(); - final int port = "https".equals(scheme)? - (Integer)getServletContext().getAttribute("datanode.https.port") - : host.getInfoPort(); - final String filename = request.getPathInfo(); - StringBuilder params = new StringBuilder(); - params.append("filename="); - params.append(filename); - if (UserGroupInformation.isSecurityEnabled()) { - String tokenString = ugi.getTokens().iterator().next().encodeToUrlString(); - params.append(JspHelper.getDelegationTokenUrlParam(tokenString)); - } else { - params.append("&ugi="); - params.append(ugi.getShortUserName()); - } - - // Add namenode address to the URL params - String nnAddr = NameNode.getHostPortString(nn.getNameNodeAddress()); - params.append(JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr)); - return new URI(scheme, null, hostname, port, servletpath, - params.toString(), null); - } - - /** Get filename from the request */ - protected String getFilename(HttpServletRequest request, - HttpServletResponse response) throws IOException { - final String filename = request.getParameter("filename"); - if (filename == null || filename.length() == 0) { - throw new IOException("Invalid filename"); - } - return filename; - } - protected UserGroupInformation getUGI(HttpServletRequest request, Configuration conf) throws IOException { return JspHelper.getUGI(getServletContext(), request, conf); diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupInputStream.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupInputStream.java index 007578f28a6..8921bc0c554 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupInputStream.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupInputStream.java @@ -21,6 +21,8 @@ import java.io.DataInputStream; import java.io.ByteArrayInputStream; import java.io.IOException; +import com.google.common.base.Preconditions; + /** * An implementation of the abstract class {@link EditLogInputStream}, * which is used to updates HDFS meta-data state on a backup node. @@ -33,6 +35,9 @@ class EditLogBackupInputStream extends EditLogInputStream { String address; // sender address private ByteBufferInputStream inner; private DataInputStream in; + private FSEditLogOp.Reader reader = null; + private FSEditLogLoader.PositionTrackingInputStream tracker = null; + private int version = 0; /** * A ByteArrayInputStream, which lets modify the underlying byte array. @@ -60,7 +65,8 @@ class EditLogBackupInputStream extends EditLogInputStream { EditLogBackupInputStream(String name) throws IOException { address = name; inner = new ByteBufferInputStream(); - in = new DataInputStream(inner); + in = null; + reader = null; } @Override // JournalStream @@ -74,18 +80,20 @@ class EditLogBackupInputStream extends EditLogInputStream { } @Override - public int available() throws IOException { - return in.available(); + public FSEditLogOp readOp() throws IOException { + Preconditions.checkState(reader != null, + "Must call setBytes() before readOp()"); + return reader.readOp(); } @Override - public int read() throws IOException { - return in.read(); + public int getVersion() throws IOException { + return this.version; } @Override - public int read(byte[] b, int off, int len) throws IOException { - return in.read(b, off, len); + public long getPosition() { + return tracker.getPos(); } @Override @@ -99,16 +107,19 @@ class EditLogBackupInputStream extends EditLogInputStream { return inner.length(); } - DataInputStream getDataInputStream() { - return in; - } - - void setBytes(byte[] newBytes) throws IOException { + void setBytes(byte[] newBytes, int version) throws IOException { inner.setData(newBytes); - in.reset(); + tracker = new FSEditLogLoader.PositionTrackingInputStream(inner); + in = new DataInputStream(tracker); + + this.version = version; + + reader = new FSEditLogOp.Reader(in, version); } void clear() throws IOException { - setBytes(null); + setBytes(null, 0); + reader = null; + this.version = 0; } } diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java index 2ae0bd038f9..532b2f2dcf4 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java @@ -21,18 +21,51 @@ package org.apache.hadoop.hdfs.server.namenode; import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.io.BufferedInputStream; +import java.io.EOFException; +import java.io.DataInputStream; +import org.apache.hadoop.hdfs.protocol.FSConstants; +import org.apache.hadoop.hdfs.server.common.Storage; +import org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation; +import org.apache.hadoop.io.IOUtils; + +import com.google.common.annotations.VisibleForTesting; /** * An implementation of the abstract class {@link EditLogInputStream}, which * reads edits from a local file. */ class EditLogFileInputStream extends EditLogInputStream { - private File file; - private FileInputStream fStream; - - EditLogFileInputStream(File name) throws IOException { + private final File file; + private final FileInputStream fStream; + private final int logVersion; + private final FSEditLogOp.Reader reader; + private final FSEditLogLoader.PositionTrackingInputStream tracker; + + /** + * Open an EditLogInputStream for the given file. + * @param name filename to open + * @throws LogHeaderCorruptException if the header is either missing or + * appears to be corrupt/truncated + * @throws IOException if an actual IO error occurs while reading the + * header + */ + EditLogFileInputStream(File name) + throws LogHeaderCorruptException, IOException { file = name; fStream = new FileInputStream(name); + + BufferedInputStream bin = new BufferedInputStream(fStream); + tracker = new FSEditLogLoader.PositionTrackingInputStream(bin); + DataInputStream in = new DataInputStream(tracker); + + try { + logVersion = readLogVersion(in); + } catch (EOFException eofe) { + throw new LogHeaderCorruptException("No header found in log"); + } + + reader = new FSEditLogOp.Reader(in, logVersion); } @Override // JournalStream @@ -46,18 +79,18 @@ class EditLogFileInputStream extends EditLogInputStream { } @Override - public int available() throws IOException { - return fStream.available(); + public FSEditLogOp readOp() throws IOException { + return reader.readOp(); } @Override - public int read() throws IOException { - return fStream.read(); + public int getVersion() throws IOException { + return logVersion; } @Override - public int read(byte[] b, int off, int len) throws IOException { - return fStream.read(b, off, len); + public long getPosition() { + return tracker.getPos(); } @Override @@ -76,4 +109,62 @@ class EditLogFileInputStream extends EditLogInputStream { return getName(); } + static FSEditLogLoader.EditLogValidation validateEditLog(File file) throws IOException { + EditLogFileInputStream in; + try { + in = new EditLogFileInputStream(file); + } catch (LogHeaderCorruptException corrupt) { + // If it's missing its header, this is equivalent to no transactions + FSImage.LOG.warn("Log at " + file + " has no valid header", + corrupt); + return new FSEditLogLoader.EditLogValidation(0, 0); + } + + try { + return FSEditLogLoader.validateEditLog(in); + } finally { + IOUtils.closeStream(in); + } + } + + /** + * Read the header of fsedit log + * @param in fsedit stream + * @return the edit log version number + * @throws IOException if error occurs + */ + @VisibleForTesting + static int readLogVersion(DataInputStream in) + throws IOException, LogHeaderCorruptException { + int logVersion; + try { + logVersion = in.readInt(); + } catch (EOFException eofe) { + throw new LogHeaderCorruptException( + "Reached EOF when reading log header"); + } + if (logVersion < FSConstants.LAYOUT_VERSION) { // future version + throw new LogHeaderCorruptException( + "Unexpected version of the file system log file: " + + logVersion + ". Current version = " + + FSConstants.LAYOUT_VERSION + "."); + } + assert logVersion <= Storage.LAST_UPGRADABLE_LAYOUT_VERSION : + "Unsupported version " + logVersion; + return logVersion; + } + + /** + * Exception indicating that the header of an edits log file is + * corrupted. This can be because the header is not present, + * or because the header data is invalid (eg claims to be + * over a newer version than the running NameNode) + */ + static class LogHeaderCorruptException extends IOException { + private static final long serialVersionUID = 1L; + + private LogHeaderCorruptException(String msg) { + super(msg); + } + } } diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogInputStream.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogInputStream.java index 49e71f9f087..52a3dd4c203 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogInputStream.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogInputStream.java @@ -17,10 +17,8 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import java.io.BufferedInputStream; -import java.io.DataInputStream; +import java.io.Closeable; import java.io.IOException; -import java.io.InputStream; /** * A generic abstract class to support reading edits log data from @@ -29,29 +27,41 @@ import java.io.InputStream; * It should stream bytes from the storage exactly as they were written * into the #{@link EditLogOutputStream}. */ -abstract class EditLogInputStream extends InputStream -implements JournalStream { - /** {@inheritDoc} */ - public abstract int available() throws IOException; - - /** {@inheritDoc} */ - public abstract int read() throws IOException; - - /** {@inheritDoc} */ - public abstract int read(byte[] b, int off, int len) throws IOException; - - /** {@inheritDoc} */ +abstract class EditLogInputStream implements JournalStream, Closeable { + /** + * Close the stream. + * @throws IOException if an error occurred while closing + */ public abstract void close() throws IOException; + /** + * Read an operation from the stream + * @return an operation from the stream or null if at end of stream + * @throws IOException if there is an error reading from the stream + */ + public abstract FSEditLogOp readOp() throws IOException; + + /** + * Get the layout version of the data in the stream. + * @return the layout version of the ops in the stream. + * @throws IOException if there is an error reading the version + */ + public abstract int getVersion() throws IOException; + + /** + * Get the "position" of in the stream. This is useful for + * debugging and operational purposes. + * + * Different stream types can have a different meaning for + * what the position is. For file streams it means the byte offset + * from the start of the file. + * + * @return the position in the stream + */ + public abstract long getPosition(); + /** * Return the size of the current edits log. */ abstract long length() throws IOException; - - /** - * Return DataInputStream based on this edit stream. - */ - DataInputStream getDataInputStream() { - return new DataInputStream(new BufferedInputStream(this)); - } } diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index 7c59bdf5722..4ad7c7e451b 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -444,8 +444,6 @@ public class FSDirectory implements Closeable { // modify file-> block and blocksMap fileNode.removeLastBlock(block); getBlockManager().removeBlockFromMap(block); - // If block is removed from blocksMap remove it from corruptReplicasMap - getBlockManager().removeFromCorruptReplicasMap(block); // write modified block locations to log fsImage.getEditLog().logOpenFile(path, fileNode); @@ -809,7 +807,7 @@ public class FSDirectory implements Closeable { * @return array of file blocks * @throws QuotaExceededException */ - Block[] setReplication(String src, short replication, int[] oldReplication) + Block[] setReplication(String src, short replication, short[] oldReplication) throws QuotaExceededException, UnresolvedLinkException { waitForReady(); Block[] fileBlocks = null; @@ -826,14 +824,10 @@ public class FSDirectory implements Closeable { Block[] unprotectedSetReplication(String src, short replication, - int[] oldReplication + short[] oldReplication ) throws QuotaExceededException, UnresolvedLinkException { assert hasWriteLock(); - if (oldReplication == null) { - oldReplication = new int[1]; - } - oldReplication[0] = -1; INode[] inodes = rootDir.getExistingPathINodes(src, true); INode inode = inodes[inodes.length - 1]; @@ -845,14 +839,17 @@ public class FSDirectory implements Closeable { return null; } INodeFile fileNode = (INodeFile)inode; - oldReplication[0] = fileNode.getReplication(); + final short oldRepl = fileNode.getReplication(); // check disk quota - long dsDelta = (replication - oldReplication[0]) * - (fileNode.diskspaceConsumed()/oldReplication[0]); + long dsDelta = (replication - oldRepl) * (fileNode.diskspaceConsumed()/oldRepl); updateCount(inodes, inodes.length-1, 0, dsDelta, true); fileNode.setReplication(replication); + + if (oldReplication != null) { + oldReplication[0] = oldRepl; + } return fileNode.getBlocks(); } @@ -1344,7 +1341,7 @@ public class FSDirectory implements Closeable { * @throws QuotaExceededException if the new count violates any quota limit * @throws FileNotFound if path does not exist. */ - public void updateSpaceConsumed(String path, long nsDelta, long dsDelta) + void updateSpaceConsumed(String path, long nsDelta, long dsDelta) throws QuotaExceededException, FileNotFoundException, UnresolvedLinkException { @@ -2075,8 +2072,9 @@ public class FSDirectory implements Closeable { size = fileNode.computeFileSize(true); replication = fileNode.getReplication(); blocksize = fileNode.getPreferredBlockSize(); - loc = getFSNamesystem().getBlockLocationsInternal( - fileNode, 0L, size, false); + loc = getFSNamesystem().getBlockManager().createLocatedBlocks( + fileNode.getBlocks(), fileNode.computeFileSize(false), + fileNode.isUnderConstruction(), 0L, size, false); if (loc==null) { loc = new LocatedBlocks(); } diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java index bf137ae9c62..495c42e45a8 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java @@ -18,10 +18,10 @@ package org.apache.hadoop.hdfs.server.namenode; import java.io.IOException; +import java.util.Collections; import java.util.Iterator; import java.util.List; -import java.util.zip.Checksum; -import java.util.zip.CheckedOutputStream; +import java.util.SortedSet; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -29,28 +29,26 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.Options; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.FSConstants; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.server.common.HdfsConstants.NamenodeRole; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import static org.apache.hadoop.hdfs.server.common.Util.now; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; -import org.apache.hadoop.hdfs.server.namenode.NNStorageRetentionManager.StoragePurger; import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics; import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Writable; +import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog; import org.apache.hadoop.security.token.delegation.DelegationKey; -import org.apache.hadoop.util.PureJavaCrc32; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.Lists; +import com.google.common.collect.Multimaps; +import com.google.common.collect.Sets; -import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.*; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.*; /** @@ -116,18 +114,6 @@ public class FSEditLog { private NNStorage storage; - private static ThreadLocal localChecksum = - new ThreadLocal() { - protected Checksum initialValue() { - return new PureJavaCrc32(); - } - }; - - /** Get a thread local checksum */ - public static Checksum getChecksum() { - return localChecksum.get(); - } - private static class TransactionId { public long txid; @@ -148,15 +134,6 @@ public class FSEditLog { this.storage = storage; metrics = NameNode.getNameNodeMetrics(); lastPrintTime = now(); - } - - /** - * Initialize the list of edit journals - */ - synchronized void initJournals() { - assert journals.isEmpty(); - Preconditions.checkState(state == State.UNINITIALIZED, - "Bad state: %s", state); for (StorageDirectory sd : storage.dirIterable(NameNodeDirType.EDITS)) { journals.add(new JournalAndStream(new FileJournalManager(sd))); @@ -174,8 +151,7 @@ public class FSEditLog { * log segment. */ synchronized void open() throws IOException { - Preconditions.checkState(state == State.UNINITIALIZED); - initJournals(); + Preconditions.checkState(state == State.BETWEEN_LOG_SEGMENTS); startLogSegment(getLastWrittenTxId() + 1, true); assert state == State.IN_SEGMENT : "Bad state: " + state; @@ -755,18 +731,64 @@ public class FSEditLog { /** * Return a manifest of what finalized edit logs are available */ - public RemoteEditLogManifest getEditLogManifest(long sinceTxId) - throws IOException { - FSImageTransactionalStorageInspector inspector = - new FSImageTransactionalStorageInspector(); - - for (StorageDirectory sd : storage.dirIterable(NameNodeDirType.EDITS)) { - inspector.inspectDirectory(sd); + public synchronized RemoteEditLogManifest getEditLogManifest( + long fromTxId) throws IOException { + // Collect RemoteEditLogs available from each FileJournalManager + List allLogs = Lists.newArrayList(); + for (JournalAndStream j : journals) { + if (j.getManager() instanceof FileJournalManager) { + FileJournalManager fjm = (FileJournalManager)j.getManager(); + try { + allLogs.addAll(fjm.getRemoteEditLogs(fromTxId)); + } catch (Throwable t) { + LOG.warn("Cannot list edit logs in " + fjm, t); + } + } } - return inspector.getEditLogManifest(sinceTxId); + // Group logs by their starting txid + ImmutableListMultimap logsByStartTxId = + Multimaps.index(allLogs, RemoteEditLog.GET_START_TXID); + long curStartTxId = fromTxId; + + List logs = Lists.newArrayList(); + while (true) { + ImmutableList logGroup = logsByStartTxId.get(curStartTxId); + if (logGroup.isEmpty()) { + // we have a gap in logs - for example because we recovered some old + // storage directory with ancient logs. Clear out any logs we've + // accumulated so far, and then skip to the next segment of logs + // after the gap. + SortedSet startTxIds = Sets.newTreeSet(logsByStartTxId.keySet()); + startTxIds = startTxIds.tailSet(curStartTxId); + if (startTxIds.isEmpty()) { + break; + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Found gap in logs at " + curStartTxId + ": " + + "not returning previous logs in manifest."); + } + logs.clear(); + curStartTxId = startTxIds.first(); + continue; + } + } + + // Find the one that extends the farthest forward + RemoteEditLog bestLog = Collections.max(logGroup); + logs.add(bestLog); + // And then start looking from after that point + curStartTxId = bestLog.getEndTxId() + 1; + } + RemoteEditLogManifest ret = new RemoteEditLogManifest(logs); + + if (LOG.isDebugEnabled()) { + LOG.debug("Generated manifest for logs since " + fromTxId + ":" + + ret); + } + return ret; } - + /** * Finalizes the current edit log and opens a new log segment. * @return the transaction id of the BEGIN_LOG_SEGMENT transaction @@ -877,8 +899,7 @@ public class FSEditLog { /** * Archive any log files that are older than the given txid. */ - public void purgeLogsOlderThan( - final long minTxIdToKeep, final StoragePurger purger) { + public void purgeLogsOlderThan(final long minTxIdToKeep) { synchronized (this) { // synchronized to prevent findbugs warning about inconsistent // synchronization. This will be JIT-ed out if asserts are @@ -892,7 +913,7 @@ public class FSEditLog { mapJournalsAndReportErrors(new JournalClosure() { @Override public void apply(JournalAndStream jas) throws IOException { - jas.manager.purgeLogsOlderThan(minTxIdToKeep, purger); + jas.manager.purgeLogsOlderThan(minTxIdToKeep); } }, "purging logs older than " + minTxIdToKeep); } @@ -1080,7 +1101,8 @@ public class FSEditLog { stream = null; } - private void abort() { + @VisibleForTesting + void abort() { if (stream == null) return; try { stream.abort(); diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java index ebe19cfb08a..db985691f63 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java @@ -19,15 +19,12 @@ package org.apache.hadoop.hdfs.server.namenode; import static org.apache.hadoop.hdfs.server.common.Util.now; -import java.io.BufferedInputStream; -import java.io.DataInputStream; import java.io.File; -import java.io.FileInputStream; import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.util.Arrays; -import java.util.zip.Checksum; +import java.util.EnumMap; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.hdfs.protocol.FSConstants; @@ -37,8 +34,7 @@ import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; import org.apache.hadoop.hdfs.server.common.Storage; -import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.LogHeader; -import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.Reader; +import org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream.LogHeaderCorruptException; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.AddCloseOp; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.CancelDelegationTokenOp; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.ClearNSQuotaOp; @@ -60,6 +56,10 @@ import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SymlinkOp; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.TimesOp; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.UpdateMasterKeyOp; import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease; +import org.apache.hadoop.hdfs.util.Holder; +import org.apache.hadoop.io.IOUtils; + +import com.google.common.base.Joiner; public class FSEditLogLoader { private final FSNamesystem fsNamesys; @@ -84,49 +84,36 @@ public class FSEditLogLoader { } int loadFSEdits(EditLogInputStream edits, boolean closeOnExit, - long expectedStartingTxId) - throws IOException { - BufferedInputStream bin = new BufferedInputStream(edits); - DataInputStream in = new DataInputStream(bin); - + long expectedStartingTxId) + throws IOException { int numEdits = 0; + int logVersion = edits.getVersion(); try { - LogHeader header = LogHeader.read(in); - numEdits = loadEditRecords( - header.logVersion, in, header.checksum, false, - expectedStartingTxId); + numEdits = loadEditRecords(logVersion, edits, false, + expectedStartingTxId); } finally { - if(closeOnExit) - in.close(); + if(closeOnExit) { + edits.close(); + } } return numEdits; } @SuppressWarnings("deprecation") - int loadEditRecords(int logVersion, DataInputStream in, - Checksum checksum, boolean closeOnExit, + int loadEditRecords(int logVersion, EditLogInputStream in, boolean closeOnExit, long expectedStartingTxId) throws IOException { FSDirectory fsDir = fsNamesys.dir; int numEdits = 0; - int numOpAdd = 0, numOpClose = 0, numOpDelete = 0, - numOpRenameOld = 0, numOpSetRepl = 0, numOpMkDir = 0, - numOpSetPerm = 0, numOpSetOwner = 0, numOpSetGenStamp = 0, - numOpTimes = 0, numOpRename = 0, numOpConcatDelete = 0, - numOpSymlink = 0, numOpGetDelegationToken = 0, - numOpRenewDelegationToken = 0, numOpCancelDelegationToken = 0, - numOpUpdateMasterKey = 0, numOpReassignLease = 0, numOpOther = 0; + EnumMap> opCounts = + new EnumMap>(FSEditLogOpCodes.class); fsNamesys.writeLock(); fsDir.writeLock(); - // Keep track of the file offsets of the last several opcodes. - // This is handy when manually recovering corrupted edits files. - PositionTrackingInputStream tracker = new PositionTrackingInputStream(in); - in = new DataInputStream(tracker); long recentOpcodeOffsets[] = new long[4]; Arrays.fill(recentOpcodeOffsets, -1); @@ -134,12 +121,10 @@ public class FSEditLogLoader { long txId = expectedStartingTxId - 1; try { - FSEditLogOp.Reader reader = new FSEditLogOp.Reader(in, logVersion, - checksum); FSEditLogOp op; - while ((op = reader.readOp()) != null) { + while ((op = in.readOp()) != null) { recentOpcodeOffsets[numEdits % recentOpcodeOffsets.length] = - tracker.getPos(); + in.getPosition(); if (LayoutVersion.supports(Feature.STORED_TXIDS, logVersion)) { long thisTxId = op.txid; if (thisTxId != txId + 1) { @@ -150,6 +135,7 @@ public class FSEditLogLoader { } numEdits++; + incrOpCount(op.opCode, opCounts); switch (op.opCode) { case OP_ADD: case OP_CLOSE: { @@ -157,8 +143,8 @@ public class FSEditLogLoader { // versions > 0 support per file replication // get name and replication - short replication - = fsNamesys.adjustReplication(addCloseOp.replication); + final short replication = fsNamesys.getBlockManager( + ).adjustReplication(addCloseOp.replication); long blockSize = addCloseOp.blockSize; BlockInfo blocks[] = new BlockInfo[addCloseOp.blocks.length]; @@ -209,7 +195,6 @@ public class FSEditLogLoader { blocks, replication, addCloseOp.mtime, addCloseOp.atime, blockSize); if (addCloseOp.opCode == FSEditLogOpCodes.OP_ADD) { - numOpAdd++; // // Replace current node with a INodeUnderConstruction. // Recreate in-memory lease record. @@ -231,24 +216,20 @@ public class FSEditLogLoader { break; } case OP_SET_REPLICATION: { - numOpSetRepl++; SetReplicationOp setReplicationOp = (SetReplicationOp)op; - short replication - = fsNamesys.adjustReplication(setReplicationOp.replication); + short replication = fsNamesys.getBlockManager().adjustReplication( + setReplicationOp.replication); fsDir.unprotectedSetReplication(setReplicationOp.path, replication, null); break; } case OP_CONCAT_DELETE: { - numOpConcatDelete++; - ConcatDeleteOp concatDeleteOp = (ConcatDeleteOp)op; fsDir.unprotectedConcat(concatDeleteOp.trg, concatDeleteOp.srcs, concatDeleteOp.timestamp); break; } case OP_RENAME_OLD: { - numOpRenameOld++; RenameOldOp renameOp = (RenameOldOp)op; HdfsFileStatus dinfo = fsDir.getFileInfo(renameOp.dst, false); fsDir.unprotectedRenameTo(renameOp.src, renameOp.dst, @@ -257,14 +238,11 @@ public class FSEditLogLoader { break; } case OP_DELETE: { - numOpDelete++; - DeleteOp deleteOp = (DeleteOp)op; fsDir.unprotectedDelete(deleteOp.path, deleteOp.timestamp); break; } case OP_MKDIR: { - numOpMkDir++; MkdirOp mkdirOp = (MkdirOp)op; PermissionStatus permissions = fsNamesys.getUpgradePermission(); if (mkdirOp.permissions != null) { @@ -276,22 +254,17 @@ public class FSEditLogLoader { break; } case OP_SET_GENSTAMP: { - numOpSetGenStamp++; SetGenstampOp setGenstampOp = (SetGenstampOp)op; fsNamesys.setGenerationStamp(setGenstampOp.genStamp); break; } case OP_SET_PERMISSIONS: { - numOpSetPerm++; - SetPermissionsOp setPermissionsOp = (SetPermissionsOp)op; fsDir.unprotectedSetPermission(setPermissionsOp.src, setPermissionsOp.permissions); break; } case OP_SET_OWNER: { - numOpSetOwner++; - SetOwnerOp setOwnerOp = (SetOwnerOp)op; fsDir.unprotectedSetOwner(setOwnerOp.src, setOwnerOp.username, setOwnerOp.groupname); @@ -320,7 +293,6 @@ public class FSEditLogLoader { break; case OP_TIMES: { - numOpTimes++; TimesOp timesOp = (TimesOp)op; fsDir.unprotectedSetTimes(timesOp.path, @@ -329,8 +301,6 @@ public class FSEditLogLoader { break; } case OP_SYMLINK: { - numOpSymlink++; - SymlinkOp symlinkOp = (SymlinkOp)op; fsDir.unprotectedSymlink(symlinkOp.path, symlinkOp.value, symlinkOp.mtime, symlinkOp.atime, @@ -338,7 +308,6 @@ public class FSEditLogLoader { break; } case OP_RENAME: { - numOpRename++; RenameOp renameOp = (RenameOp)op; HdfsFileStatus dinfo = fsDir.getFileInfo(renameOp.dst, false); @@ -348,7 +317,6 @@ public class FSEditLogLoader { break; } case OP_GET_DELEGATION_TOKEN: { - numOpGetDelegationToken++; GetDelegationTokenOp getDelegationTokenOp = (GetDelegationTokenOp)op; @@ -358,8 +326,6 @@ public class FSEditLogLoader { break; } case OP_RENEW_DELEGATION_TOKEN: { - numOpRenewDelegationToken++; - RenewDelegationTokenOp renewDelegationTokenOp = (RenewDelegationTokenOp)op; fsNamesys.getDelegationTokenSecretManager() @@ -368,8 +334,6 @@ public class FSEditLogLoader { break; } case OP_CANCEL_DELEGATION_TOKEN: { - numOpCancelDelegationToken++; - CancelDelegationTokenOp cancelDelegationTokenOp = (CancelDelegationTokenOp)op; fsNamesys.getDelegationTokenSecretManager() @@ -378,14 +342,12 @@ public class FSEditLogLoader { break; } case OP_UPDATE_MASTER_KEY: { - numOpUpdateMasterKey++; UpdateMasterKeyOp updateMasterKeyOp = (UpdateMasterKeyOp)op; fsNamesys.getDelegationTokenSecretManager() .updatePersistedMasterKey(updateMasterKeyOp.key); break; } case OP_REASSIGN_LEASE: { - numOpReassignLease++; ReassignLeaseOp reassignLeaseOp = (ReassignLeaseOp)op; Lease lease = fsNamesys.leaseManager.getLease( @@ -400,17 +362,16 @@ public class FSEditLogLoader { case OP_START_LOG_SEGMENT: case OP_END_LOG_SEGMENT: { // no data in here currently. - numOpOther++; break; } case OP_DATANODE_ADD: case OP_DATANODE_REMOVE: - numOpOther++; break; default: throw new IOException("Invalid operation read " + op.opCode); } } + } catch (IOException ex) { check203UpgradeFailure(logVersion, ex); } finally { @@ -421,7 +382,7 @@ public class FSEditLogLoader { // Catch Throwable because in the case of a truly corrupt edits log, any // sort of error might be thrown (NumberFormat, NullPointer, EOF, etc.) StringBuilder sb = new StringBuilder(); - sb.append("Error replaying edit log at offset " + tracker.getPos()); + sb.append("Error replaying edit log at offset " + in.getPosition()); if (recentOpcodeOffsets[0] != -1) { Arrays.sort(recentOpcodeOffsets); sb.append("\nRecent opcode offsets:"); @@ -439,26 +400,31 @@ public class FSEditLogLoader { fsNamesys.writeUnlock(); } if (FSImage.LOG.isDebugEnabled()) { - FSImage.LOG.debug("numOpAdd = " + numOpAdd + " numOpClose = " + numOpClose - + " numOpDelete = " + numOpDelete - + " numOpRenameOld = " + numOpRenameOld - + " numOpSetRepl = " + numOpSetRepl + " numOpMkDir = " + numOpMkDir - + " numOpSetPerm = " + numOpSetPerm - + " numOpSetOwner = " + numOpSetOwner - + " numOpSetGenStamp = " + numOpSetGenStamp - + " numOpTimes = " + numOpTimes - + " numOpConcatDelete = " + numOpConcatDelete - + " numOpRename = " + numOpRename - + " numOpGetDelegationToken = " + numOpGetDelegationToken - + " numOpRenewDelegationToken = " + numOpRenewDelegationToken - + " numOpCancelDelegationToken = " + numOpCancelDelegationToken - + " numOpUpdateMasterKey = " + numOpUpdateMasterKey - + " numOpReassignLease = " + numOpReassignLease - + " numOpOther = " + numOpOther); + dumpOpCounts(opCounts); } return numEdits; } + + private static void dumpOpCounts( + EnumMap> opCounts) { + StringBuilder sb = new StringBuilder(); + sb.append("Summary of operations loaded from edit log:\n "); + Joiner.on("\n ").withKeyValueSeparator("=").appendTo(sb, opCounts); + FSImage.LOG.debug(sb.toString()); + } + + private void incrOpCount(FSEditLogOpCodes opCode, + EnumMap> opCounts) { + Holder holder = opCounts.get(opCode); + if (holder == null) { + holder = new Holder(1); + opCounts.put(opCode, holder); + } else { + holder.held++; + } + } + /** * Throw appropriate exception during upgrade from 203, when editlog loading * could fail due to opcode conflicts. @@ -480,49 +446,50 @@ public class FSEditLogLoader { } } + static EditLogValidation validateEditLog(File file) throws IOException { + EditLogFileInputStream in; + try { + in = new EditLogFileInputStream(file); + } catch (LogHeaderCorruptException corrupt) { + // If it's missing its header, this is equivalent to no transactions + FSImage.LOG.warn("Log at " + file + " has no valid header", + corrupt); + return new EditLogValidation(0, 0); + } + + try { + return validateEditLog(in); + } finally { + IOUtils.closeStream(in); + } + } + /** - * Return the number of valid transactions in the file. If the file is + * Return the number of valid transactions in the stream. If the stream is * truncated during the header, returns a value indicating that there are - * 0 valid transactions. - * @throws IOException if the file cannot be read due to an IO error (eg + * 0 valid transactions. This reads through the stream but does not close + * it. + * @throws IOException if the stream cannot be read due to an IO error (eg * if the log does not exist) */ - static EditLogValidation validateEditLog(File f) throws IOException { - FileInputStream fis = new FileInputStream(f); + static EditLogValidation validateEditLog(EditLogInputStream in) { + long numValid = 0; + long lastPos = 0; try { - PositionTrackingInputStream tracker = new PositionTrackingInputStream( - new BufferedInputStream(fis)); - DataInputStream dis = new DataInputStream(tracker); - LogHeader header; - try { - header = LogHeader.read(dis); - } catch (Throwable t) { - FSImage.LOG.debug("Unable to read header from " + f + - " -> no valid transactions in this file."); - return new EditLogValidation(0, 0); - } - - Reader reader = new FSEditLogOp.Reader(dis, header.logVersion, header.checksum); - long numValid = 0; - long lastPos = 0; - try { - while (true) { - lastPos = tracker.getPos(); - if (reader.readOp() == null) { - break; - } - numValid++; + while (true) { + lastPos = in.getPosition(); + if (in.readOp() == null) { + break; } - } catch (Throwable t) { - // Catch Throwable and not just IOE, since bad edits may generate - // NumberFormatExceptions, AssertionErrors, OutOfMemoryErrors, etc. - FSImage.LOG.debug("Caught exception after reading " + numValid + - " ops from " + f + " while determining its valid length.", t); + numValid++; } - return new EditLogValidation(lastPos, numValid); - } finally { - fis.close(); + } catch (Throwable t) { + // Catch Throwable and not just IOE, since bad edits may generate + // NumberFormatExceptions, AssertionErrors, OutOfMemoryErrors, etc. + FSImage.LOG.debug("Caught exception after reading " + numValid + + " ops from " + in + " while determining its valid length.", t); } + return new EditLogValidation(lastPos, numValid); } static class EditLogValidation { @@ -536,9 +503,9 @@ public class FSEditLogLoader { } /** - * Stream wrapper that keeps track of the current file position. + * Stream wrapper that keeps track of the current stream position. */ - private static class PositionTrackingInputStream extends FilterInputStream { + static class PositionTrackingInputStream extends FilterInputStream { private long curPos = 0; private long markPos = -1; @@ -582,4 +549,5 @@ public class FSEditLogLoader { return curPos; } } + } diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java index 00ce353b9ac..6529c876c0b 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.common.GenerationStamp; import org.apache.hadoop.hdfs.server.common.Storage; +import org.apache.hadoop.util.PureJavaCrc32; import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.*; import org.apache.hadoop.security.token.delegation.DelegationKey; @@ -1323,71 +1324,17 @@ public abstract class FSEditLogOp { return longWritable.get(); } } - - /** - * Class to encapsulate the header at the top of a log file. - */ - static class LogHeader { - final int logVersion; - final Checksum checksum; - - public LogHeader(int logVersion, Checksum checksum) { - this.logVersion = logVersion; - this.checksum = checksum; - } - - static LogHeader read(DataInputStream in) throws IOException { - int logVersion = 0; - - logVersion = FSEditLogOp.LogHeader.readLogVersion(in); - Checksum checksum = null; - if (LayoutVersion.supports(Feature.EDITS_CHESKUM, logVersion)) { - checksum = FSEditLog.getChecksum(); - } - return new LogHeader(logVersion, checksum); - } - - /** - * Read the header of fsedit log - * @param in fsedit stream - * @return the edit log version number - * @throws IOException if error occurs - */ - private static int readLogVersion(DataInputStream in) throws IOException { - int logVersion = 0; - // Read log file version. Could be missing. - in.mark(4); - // If edits log is greater than 2G, available method will return negative - // numbers, so we avoid having to call available - boolean available = true; - try { - logVersion = in.readByte(); - } catch (EOFException e) { - available = false; - } - if (available) { - in.reset(); - logVersion = in.readInt(); - if (logVersion < FSConstants.LAYOUT_VERSION) // future version - throw new IOException( - "Unexpected version of the file system log file: " - + logVersion + ". Current version = " - + FSConstants.LAYOUT_VERSION + "."); - } - assert logVersion <= Storage.LAST_UPGRADABLE_LAYOUT_VERSION : - "Unsupported version " + logVersion; - return logVersion; - } - } /** * Class for writing editlog ops */ public static class Writer { private final DataOutputBuffer buf; + private final Checksum checksum; public Writer(DataOutputBuffer out) { this.buf = out; + this.checksum = new PureJavaCrc32(); } /** @@ -1402,7 +1349,6 @@ public abstract class FSEditLogOp { buf.writeLong(op.txid); op.writeFields(buf); int end = buf.getLength(); - Checksum checksum = FSEditLog.getChecksum(); checksum.reset(); checksum.update(buf.getData(), start, end-start); int sum = (int)checksum.getValue(); @@ -1422,19 +1368,22 @@ public abstract class FSEditLogOp { * Construct the reader * @param in The stream to read from. * @param logVersion The version of the data coming from the stream. - * @param checksum Checksum being used with input stream. */ @SuppressWarnings("deprecation") - public Reader(DataInputStream in, int logVersion, - Checksum checksum) { - if (checksum != null) { + public Reader(DataInputStream in, int logVersion) { + this.logVersion = logVersion; + if (LayoutVersion.supports(Feature.EDITS_CHESKUM, logVersion)) { + this.checksum = new PureJavaCrc32(); + } else { + this.checksum = null; + } + + if (this.checksum != null) { this.in = new DataInputStream( - new CheckedInputStream(in, checksum)); + new CheckedInputStream(in, this.checksum)); } else { this.in = in; } - this.logVersion = logVersion; - this.checksum = checksum; } /** diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java index 993dd8cd44e..8b259018f19 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java @@ -137,10 +137,6 @@ public class FSImage implements Closeable { FSImage.getCheckpointEditsDirs(conf, null)); storage = new NNStorage(conf, imageDirs, editsDirs); - if (ns != null) { - storage.setUpgradeManager(ns.upgradeManager); - } - if(conf.getBoolean(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_RESTORE_KEY, DFSConfigKeys.DFS_NAMENODE_NAME_DIR_RESTORE_DEFAULT)) { storage.setRestoreFailedStorage(true); diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java index 5bad6be91dd..453985d917a 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java @@ -330,7 +330,7 @@ class FSImageFormat { int imgVersion = getLayoutVersion(); short replication = in.readShort(); - replication = namesystem.adjustReplication(replication); + replication = namesystem.getBlockManager().adjustReplication(replication); modificationTime = in.readLong(); if (LayoutVersion.supports(Feature.FILE_ACCESS_TIME, imgVersion)) { atime = in.readLong(); diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageStorageInspector.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageStorageInspector.java index 6249f2f5d5d..65bfa0ac556 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageStorageInspector.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageStorageInspector.java @@ -96,4 +96,36 @@ abstract class FSImageStorageInspector { return sb.toString(); } } + + /** + * Record of an image that has been located and had its filename parsed. + */ + static class FSImageFile { + final StorageDirectory sd; + final long txId; + private final File file; + + FSImageFile(StorageDirectory sd, File file, long txId) { + assert txId >= 0 : "Invalid txid on " + file +": " + txId; + + this.sd = sd; + this.txId = txId; + this.file = file; + } + + File getFile() { + return file; + } + + public long getCheckpointTxId() { + return txId; + } + + @Override + public String toString() { + return String.format("FSImageFile(file=%s, cpktTxId=%019d)", + file.toString(), txId); + } + } + } diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java index 8a2c2aa4db9..0814a140b5e 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java @@ -37,9 +37,9 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdfs.protocol.FSConstants; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; -import org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile; +import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; @@ -54,17 +54,13 @@ class FSImageTransactionalStorageInspector extends FSImageStorageInspector { private boolean needToSave = false; private boolean isUpgradeFinalized = true; - List foundImages = new ArrayList(); - List foundEditLogs = new ArrayList(); + List foundImages = new ArrayList(); + List foundEditLogs = new ArrayList(); SortedMap logGroups = new TreeMap(); long maxSeenTxId = 0; private static final Pattern IMAGE_REGEX = Pattern.compile( NameNodeFile.IMAGE.getName() + "_(\\d+)"); - private static final Pattern EDITS_REGEX = Pattern.compile( - NameNodeFile.EDITS.getName() + "_(\\d+)-(\\d+)"); - private static final Pattern EDITS_INPROGRESS_REGEX = Pattern.compile( - NameNodeFile.EDITS_INPROGRESS.getName() + "_(\\d+)"); @Override public void inspectDirectory(StorageDirectory sd) throws IOException { @@ -95,7 +91,7 @@ class FSImageTransactionalStorageInspector extends FSImageStorageInspector { if (sd.getStorageDirType().isOfType(NameNodeDirType.IMAGE)) { try { long txid = Long.valueOf(imageMatch.group(1)); - foundImages.add(new FoundFSImage(sd, f, txid)); + foundImages.add(new FSImageFile(sd, f, txid)); } catch (NumberFormatException nfe) { LOG.error("Image file " + f + " has improperly formatted " + "transaction ID"); @@ -117,9 +113,10 @@ class FSImageTransactionalStorageInspector extends FSImageStorageInspector { LOG.warn("Unable to determine the max transaction ID seen by " + sd, ioe); } - List editLogs = matchEditLogs(filesInStorage); + List editLogs + = FileJournalManager.matchEditLogs(filesInStorage); if (sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) { - for (FoundEditLog log : editLogs) { + for (EditLogFile log : editLogs) { addEditLog(log); } } else if (!editLogs.isEmpty()){ @@ -133,47 +130,12 @@ class FSImageTransactionalStorageInspector extends FSImageStorageInspector { isUpgradeFinalized = isUpgradeFinalized && !sd.getPreviousDir().exists(); } - static List matchEditLogs(File[] filesInStorage) { - List ret = Lists.newArrayList(); - for (File f : filesInStorage) { - String name = f.getName(); - // Check for edits - Matcher editsMatch = EDITS_REGEX.matcher(name); - if (editsMatch.matches()) { - try { - long startTxId = Long.valueOf(editsMatch.group(1)); - long endTxId = Long.valueOf(editsMatch.group(2)); - ret.add(new FoundEditLog(f, startTxId, endTxId)); - } catch (NumberFormatException nfe) { - LOG.error("Edits file " + f + " has improperly formatted " + - "transaction ID"); - // skip - } - } - - // Check for in-progress edits - Matcher inProgressEditsMatch = EDITS_INPROGRESS_REGEX.matcher(name); - if (inProgressEditsMatch.matches()) { - try { - long startTxId = Long.valueOf(inProgressEditsMatch.group(1)); - ret.add( - new FoundEditLog(f, startTxId, FoundEditLog.UNKNOWN_END)); - } catch (NumberFormatException nfe) { - LOG.error("In-progress edits file " + f + " has improperly " + - "formatted transaction ID"); - // skip - } - } - } - return ret; - } - - private void addEditLog(FoundEditLog foundEditLog) { + private void addEditLog(EditLogFile foundEditLog) { foundEditLogs.add(foundEditLog); - LogGroup group = logGroups.get(foundEditLog.startTxId); + LogGroup group = logGroups.get(foundEditLog.getFirstTxId()); if (group == null) { - group = new LogGroup(foundEditLog.startTxId); - logGroups.put(foundEditLog.startTxId, group); + group = new LogGroup(foundEditLog.getFirstTxId()); + logGroups.put(foundEditLog.getFirstTxId(), group); } group.add(foundEditLog); } @@ -191,9 +153,9 @@ class FSImageTransactionalStorageInspector extends FSImageStorageInspector { * * Returns null if no images were found. */ - FoundFSImage getLatestImage() { - FoundFSImage ret = null; - for (FoundFSImage img : foundImages) { + FSImageFile getLatestImage() { + FSImageFile ret = null; + for (FSImageFile img : foundImages) { if (ret == null || img.txId > ret.txId) { ret = img; } @@ -201,11 +163,11 @@ class FSImageTransactionalStorageInspector extends FSImageStorageInspector { return ret; } - public List getFoundImages() { + public List getFoundImages() { return ImmutableList.copyOf(foundImages); } - public List getFoundEditLogs() { + public List getEditLogFiles() { return ImmutableList.copyOf(foundEditLogs); } @@ -215,7 +177,7 @@ class FSImageTransactionalStorageInspector extends FSImageStorageInspector { throw new FileNotFoundException("No valid image files found"); } - FoundFSImage recoveryImage = getLatestImage(); + FSImageFile recoveryImage = getLatestImage(); LogLoadPlan logPlan = createLogLoadPlan(recoveryImage.txId, Long.MAX_VALUE); return new TransactionalLoadPlan(recoveryImage, @@ -233,7 +195,7 @@ class FSImageTransactionalStorageInspector extends FSImageStorageInspector { LogLoadPlan createLogLoadPlan(long sinceTxId, long maxStartTxId) throws IOException { long expectedTxId = sinceTxId + 1; - List recoveryLogs = new ArrayList(); + List recoveryLogs = new ArrayList(); SortedMap tailGroups = logGroups.tailMap(expectedTxId); if (logGroups.size() > tailGroups.size()) { @@ -306,22 +268,6 @@ class FSImageTransactionalStorageInspector extends FSImageStorageInspector { return needToSave; } - - RemoteEditLogManifest getEditLogManifest(long sinceTxId) { - List logs = Lists.newArrayList(); - for (LogGroup g : logGroups.values()) { - if (!g.hasFinalized) continue; - - FoundEditLog fel = g.getBestNonCorruptLog(); - if (fel.getLastTxId() < sinceTxId) continue; - - logs.add(new RemoteEditLog(fel.getStartTxId(), - fel.getLastTxId())); - } - - return new RemoteEditLogManifest(logs); - } - /** * A group of logs that all start at the same txid. * @@ -330,7 +276,7 @@ class FSImageTransactionalStorageInspector extends FSImageStorageInspector { */ static class LogGroup { long startTxId; - List logs = new ArrayList();; + List logs = new ArrayList();; private Set endTxIds = new TreeSet(); private boolean hasInProgress = false; private boolean hasFinalized = false; @@ -339,15 +285,15 @@ class FSImageTransactionalStorageInspector extends FSImageStorageInspector { this.startTxId = startTxId; } - FoundEditLog getBestNonCorruptLog() { + EditLogFile getBestNonCorruptLog() { // First look for non-corrupt finalized logs - for (FoundEditLog log : logs) { + for (EditLogFile log : logs) { if (!log.isCorrupt() && !log.isInProgress()) { return log; } } // Then look for non-corrupt in-progress logs - for (FoundEditLog log : logs) { + for (EditLogFile log : logs) { if (!log.isCorrupt()) { return log; } @@ -364,7 +310,7 @@ class FSImageTransactionalStorageInspector extends FSImageStorageInspector { * @return true if we can determine the last txid in this log group. */ boolean hasKnownLastTxId() { - for (FoundEditLog log : logs) { + for (EditLogFile log : logs) { if (!log.isInProgress()) { return true; } @@ -378,24 +324,24 @@ class FSImageTransactionalStorageInspector extends FSImageStorageInspector { * {@see #hasKnownLastTxId()} */ long getLastTxId() { - for (FoundEditLog log : logs) { + for (EditLogFile log : logs) { if (!log.isInProgress()) { - return log.lastTxId; + return log.getLastTxId(); } } throw new IllegalStateException("LogGroup only has in-progress logs"); } - void add(FoundEditLog log) { - assert log.getStartTxId() == startTxId; + void add(EditLogFile log) { + assert log.getFirstTxId() == startTxId; logs.add(log); if (log.isInProgress()) { hasInProgress = true; } else { hasFinalized = true; - endTxIds.add(log.lastTxId); + endTxIds.add(log.getLastTxId()); } } @@ -422,7 +368,7 @@ class FSImageTransactionalStorageInspector extends FSImageStorageInspector { * The in-progress logs in this case should be considered corrupt. */ private void planMixedLogRecovery() throws IOException { - for (FoundEditLog log : logs) { + for (EditLogFile log : logs) { if (log.isInProgress()) { LOG.warn("Log at " + log.getFile() + " is in progress, but " + "other logs starting at the same txid " + startTxId + @@ -446,7 +392,7 @@ class FSImageTransactionalStorageInspector extends FSImageStorageInspector { "crash)"); if (logs.size() == 1) { // Only one log, it's our only choice! - FoundEditLog log = logs.get(0); + EditLogFile log = logs.get(0); if (log.validateLog().numTransactions == 0) { // If it has no transactions, we should consider it corrupt just // to be conservative. @@ -459,7 +405,7 @@ class FSImageTransactionalStorageInspector extends FSImageStorageInspector { } long maxValidTxnCount = Long.MIN_VALUE; - for (FoundEditLog log : logs) { + for (EditLogFile log : logs) { long validTxnCount = log.validateLog().numTransactions; LOG.warn(" Log " + log.getFile() + " valid txns=" + validTxnCount + @@ -467,7 +413,7 @@ class FSImageTransactionalStorageInspector extends FSImageStorageInspector { maxValidTxnCount = Math.max(maxValidTxnCount, validTxnCount); } - for (FoundEditLog log : logs) { + for (EditLogFile log : logs) { long txns = log.validateLog().numTransactions; if (txns < maxValidTxnCount) { LOG.warn("Marking log at " + log.getFile() + " as corrupt since " + @@ -499,7 +445,7 @@ class FSImageTransactionalStorageInspector extends FSImageStorageInspector { } void recover() throws IOException { - for (FoundEditLog log : logs) { + for (EditLogFile log : logs) { if (log.isCorrupt()) { log.moveAsideCorruptFile(); } else if (log.isInProgress()) { @@ -508,131 +454,12 @@ class FSImageTransactionalStorageInspector extends FSImageStorageInspector { } } } - - /** - * Record of an image that has been located and had its filename parsed. - */ - static class FoundFSImage { - final StorageDirectory sd; - final long txId; - private final File file; - - FoundFSImage(StorageDirectory sd, File file, long txId) { - assert txId >= 0 : "Invalid txid on " + file +": " + txId; - - this.sd = sd; - this.txId = txId; - this.file = file; - } - - File getFile() { - return file; - } - - public long getTxId() { - return txId; - } - - @Override - public String toString() { - return file.toString(); - } - } - /** - * Record of an edit log that has been located and had its filename parsed. - */ - static class FoundEditLog { - File file; - final long startTxId; - long lastTxId; - - private EditLogValidation cachedValidation = null; - private boolean isCorrupt = false; - - static final long UNKNOWN_END = -1; - - FoundEditLog(File file, - long startTxId, long endTxId) { - assert endTxId == UNKNOWN_END || endTxId >= startTxId; - assert startTxId > 0; - assert file != null; - - this.startTxId = startTxId; - this.lastTxId = endTxId; - this.file = file; - } - - public void finalizeLog() throws IOException { - long numTransactions = validateLog().numTransactions; - long lastTxId = startTxId + numTransactions - 1; - File dst = new File(file.getParentFile(), - NNStorage.getFinalizedEditsFileName(startTxId, lastTxId)); - LOG.info("Finalizing edits log " + file + " by renaming to " - + dst.getName()); - if (!file.renameTo(dst)) { - throw new IOException("Couldn't finalize log " + - file + " to " + dst); - } - this.lastTxId = lastTxId; - file = dst; - } - - long getStartTxId() { - return startTxId; - } - - long getLastTxId() { - return lastTxId; - } - - EditLogValidation validateLog() throws IOException { - if (cachedValidation == null) { - cachedValidation = FSEditLogLoader.validateEditLog(file); - } - return cachedValidation; - } - - boolean isInProgress() { - return (lastTxId == UNKNOWN_END); - } - - File getFile() { - return file; - } - - void markCorrupt() { - isCorrupt = true; - } - - boolean isCorrupt() { - return isCorrupt; - } - - void moveAsideCorruptFile() throws IOException { - assert isCorrupt; - - File src = file; - File dst = new File(src.getParent(), src.getName() + ".corrupt"); - boolean success = src.renameTo(dst); - if (!success) { - throw new IOException( - "Couldn't rename corrupt log " + src + " to " + dst); - } - file = dst; - } - - @Override - public String toString() { - return file.toString(); - } - } - static class TransactionalLoadPlan extends LoadPlan { - final FoundFSImage image; + final FSImageFile image; final LogLoadPlan logPlan; - public TransactionalLoadPlan(FoundFSImage image, + public TransactionalLoadPlan(FSImageFile image, LogLoadPlan logPlan) { super(); this.image = image; @@ -662,10 +489,10 @@ class FSImageTransactionalStorageInspector extends FSImageStorageInspector { } static class LogLoadPlan { - final List editLogs; + final List editLogs; final List logGroupsToRecover; - LogLoadPlan(List editLogs, + LogLoadPlan(List editLogs, List logGroupsToRecover) { this.editLogs = editLogs; this.logGroupsToRecover = logGroupsToRecover; @@ -679,7 +506,7 @@ class FSImageTransactionalStorageInspector extends FSImageStorageInspector { public List getEditsFiles() { List ret = new ArrayList(); - for (FoundEditLog log : editLogs) { + for (EditLogFile log : editLogs) { ret.add(log.getFile()); } return ret; diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index ffede2dfa88..d4cb0f8ae51 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -42,7 +42,6 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -69,35 +68,36 @@ import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; -import org.apache.hadoop.hdfs.DFSConfigKeys; +import static org.apache.hadoop.hdfs.DFSConfigKeys.*; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException; import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.FSConstants; +import org.apache.hadoop.hdfs.protocol.FSConstants.DatanodeReportType; +import org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction; +import org.apache.hadoop.hdfs.protocol.FSConstants.UpgradeAction; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException; -import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException; import org.apache.hadoop.hdfs.protocol.datatransfer.ReplaceDatanodeOnFailure; import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager; +import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager.AccessMode; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; -import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; -import org.apache.hadoop.hdfs.server.blockmanagement.UnderReplicatedBlocks; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStatistics; import org.apache.hadoop.hdfs.server.common.GenerationStamp; import org.apache.hadoop.hdfs.server.common.HdfsConstants.BlockUCState; import org.apache.hadoop.hdfs.server.common.HdfsConstants.NamenodeRole; @@ -107,15 +107,13 @@ import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport; import org.apache.hadoop.hdfs.server.common.Util; import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease; import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean; -import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations; import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; -import org.apache.hadoop.hdfs.server.protocol.KeyUpdateCommand; import org.apache.hadoop.hdfs.server.protocol.NamenodeCommand; import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; -import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; import org.apache.hadoop.hdfs.server.protocol.UpgradeCommand; +import org.apache.hadoop.hdfs.util.RwLock; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.Server; @@ -151,8 +149,8 @@ import org.mortbay.util.ajax.JSON; ***************************************************/ @InterfaceAudience.Private @Metrics(context="dfs") -public class FSNamesystem implements FSConstants, FSNamesystemMBean, - FSClusterStats, NameNodeMXBean { +public class FSNamesystem implements RwLock, FSClusterStats, + FSNamesystemMBean, NameNodeMXBean { static final Log LOG = LogFactory.getLog(FSNamesystem.class); private static final ThreadLocal auditBuffer = @@ -207,9 +205,6 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, private PermissionStatus defaultPermission; // FSNamesystemMetrics counter variables @Metric private MutableCounterInt expiredHeartbeats; - private long capacityTotal = 0L, capacityUsed = 0L, capacityRemaining = 0L; - private long blockPoolUsed = 0L; - private int totalLoad = 0; // Scan interval is not configurable. private static final long DELEGATION_TOKEN_REMOVER_SCAN_INTERVAL = @@ -219,27 +214,16 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, // // Stores the correct file name hierarchy // - public FSDirectory dir; + FSDirectory dir; private BlockManager blockManager; - + private DatanodeStatistics datanodeStatistics; + // Block pool ID used by this namenode - String blockPoolId; + private String blockPoolId; - /** - * Stores a subset of datanodeMap, containing nodes that are considered alive. - * The HeartbeatMonitor periodically checks for out-dated entries, - * and removes them from the list. - */ - public ArrayList heartbeats = new ArrayList(); + LeaseManager leaseManager = new LeaseManager(this); - public LeaseManager leaseManager = new LeaseManager(this); - - // - // Threaded object that checks to see if we have been - // getting heartbeats from all clients. - // - Daemon hbthread = null; // HeartbeatMonitor thread - public Daemon lmthread = null; // LeaseMonitor thread + Daemon lmthread = null; // LeaseMonitor thread Daemon smmthread = null; // SafeModeMonitor thread Daemon nnrmthread = null; // NamenodeResourceMonitor thread @@ -248,9 +232,6 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, private volatile boolean fsRunning = true; long systemStart = 0; - // heartbeatRecheckInterval is how often namenode checks for expired datanodes - private long heartbeatRecheckInterval; - //resourceRecheckInterval is how often namenode checks for the disk space availability private long resourceRecheckInterval; @@ -296,13 +277,14 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, */ private void initialize(Configuration conf, FSImage fsImage) throws IOException { - resourceRecheckInterval = - conf.getLong(DFSConfigKeys.DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY, - DFSConfigKeys.DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_DEFAULT); + resourceRecheckInterval = conf.getLong( + DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY, + DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_DEFAULT); nnResourceChecker = new NameNodeResourceChecker(conf); checkAvailableResources(); this.systemStart = now(); this.blockManager = new BlockManager(this, conf); + this.datanodeStatistics = blockManager.getDatanodeManager().getDatanodeStatistics(); this.fsLock = new ReentrantReadWriteLock(true); // fair locking setConfigurationParameters(conf); dtSecretManager = createDelegationTokenSecretManager(conf); @@ -331,26 +313,28 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, * Activate FSNamesystem daemons. */ void activate(Configuration conf) throws IOException { - setBlockTotal(); - blockManager.activate(conf); - this.hbthread = new Daemon(new HeartbeatMonitor()); - this.lmthread = new Daemon(leaseManager.new Monitor()); + writeLock(); + try { + setBlockTotal(); + blockManager.activate(conf); + + this.lmthread = new Daemon(leaseManager.new Monitor()); + lmthread.start(); + this.nnrmthread = new Daemon(new NameNodeResourceMonitor()); + nnrmthread.start(); + } finally { + writeUnlock(); + } - hbthread.start(); - lmthread.start(); - - this.nnrmthread = new Daemon(new NameNodeResourceMonitor()); - nnrmthread.start(); - registerMXBean(); DefaultMetricsSystem.instance().register(this); } public static Collection getNamespaceDirs(Configuration conf) { - return getStorageDirs(conf, DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY); + return getStorageDirs(conf, DFS_NAMENODE_NAME_DIR_KEY); } - public static Collection getStorageDirs(Configuration conf, + private static Collection getStorageDirs(Configuration conf, String propertyName) { Collection dirNames = conf.getTrimmedStringCollection(propertyName); StartupOption startOpt = NameNode.getStartupOption(conf); @@ -381,35 +365,35 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, } public static Collection getNamespaceEditsDirs(Configuration conf) { - return getStorageDirs(conf, DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY); + return getStorageDirs(conf, DFS_NAMENODE_EDITS_DIR_KEY); } - // utility methods to acquire and release read lock and write lock + @Override public void readLock() { this.fsLock.readLock().lock(); } - + @Override public void readUnlock() { this.fsLock.readLock().unlock(); } - + @Override public void writeLock() { this.fsLock.writeLock().lock(); } - + @Override public void writeUnlock() { this.fsLock.writeLock().unlock(); } - + @Override public boolean hasWriteLock() { return this.fsLock.isWriteLockedByCurrentThread(); } - - boolean hasReadLock() { + @Override + public boolean hasReadLock() { return this.fsLock.getReadHoldCount() > 0; } - - boolean hasReadOrWriteLock() { + @Override + public boolean hasReadOrWriteLock() { return hasReadLock() || hasWriteLock(); } @@ -453,33 +437,30 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, LOG.info("fsOwner=" + fsOwner); - this.supergroup = conf.get(DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_KEY, - DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT); - this.isPermissionEnabled = conf.getBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, - DFSConfigKeys.DFS_PERMISSIONS_ENABLED_DEFAULT); + this.supergroup = conf.get(DFS_PERMISSIONS_SUPERUSERGROUP_KEY, + DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT); + this.isPermissionEnabled = conf.getBoolean(DFS_PERMISSIONS_ENABLED_KEY, + DFS_PERMISSIONS_ENABLED_DEFAULT); LOG.info("supergroup=" + supergroup); LOG.info("isPermissionEnabled=" + isPermissionEnabled); - short filePermission = (short)conf.getInt(DFSConfigKeys.DFS_NAMENODE_UPGRADE_PERMISSION_KEY, - DFSConfigKeys.DFS_NAMENODE_UPGRADE_PERMISSION_DEFAULT); + short filePermission = (short)conf.getInt(DFS_NAMENODE_UPGRADE_PERMISSION_KEY, + DFS_NAMENODE_UPGRADE_PERMISSION_DEFAULT); this.defaultPermission = PermissionStatus.createImmutable( fsOwner.getShortUserName(), supergroup, new FsPermission(filePermission)); - - this.heartbeatRecheckInterval = conf.getInt( - DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, - DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_DEFAULT); // 5 minutes this.serverDefaults = new FsServerDefaults( - conf.getLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE), - conf.getInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, DEFAULT_BYTES_PER_CHECKSUM), - conf.getInt(DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_KEY, DEFAULT_WRITE_PACKET_SIZE), - (short) conf.getInt(DFSConfigKeys.DFS_REPLICATION_KEY, DEFAULT_REPLICATION_FACTOR), - conf.getInt("io.file.buffer.size", DEFAULT_FILE_BUFFER_SIZE)); - this.maxFsObjects = conf.getLong(DFSConfigKeys.DFS_NAMENODE_MAX_OBJECTS_KEY, - DFSConfigKeys.DFS_NAMENODE_MAX_OBJECTS_DEFAULT); + conf.getLong(DFS_BLOCK_SIZE_KEY, DFS_BLOCK_SIZE_DEFAULT), + conf.getInt(DFS_BYTES_PER_CHECKSUM_KEY, DFS_BYTES_PER_CHECKSUM_DEFAULT), + conf.getInt(DFS_CLIENT_WRITE_PACKET_SIZE_KEY, DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT), + (short) conf.getInt(DFS_REPLICATION_KEY, DFS_REPLICATION_DEFAULT), + conf.getInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT)); + + this.maxFsObjects = conf.getLong(DFS_NAMENODE_MAX_OBJECTS_KEY, + DFS_NAMENODE_MAX_OBJECTS_DEFAULT); - this.accessTimePrecision = conf.getLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, 0); - this.supportAppends = conf.getBoolean(DFSConfigKeys.DFS_SUPPORT_APPEND_KEY, - DFSConfigKeys.DFS_SUPPORT_APPEND_DEFAULT); + this.accessTimePrecision = conf.getLong(DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, 0); + this.supportAppends = conf.getBoolean(DFS_SUPPORT_APPEND_KEY, + DFS_SUPPORT_APPEND_DEFAULT); this.dtpReplaceDatanodeOnFailure = ReplaceDatanodeOnFailure.get(conf); } @@ -497,7 +478,8 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, try { return new NamespaceInfo(dir.fsImage.getStorage().getNamespaceID(), getClusterId(), getBlockPoolId(), - dir.fsImage.getStorage().getCTime(), getDistributedUpgradeVersion()); + dir.fsImage.getStorage().getCTime(), + upgradeManager.getUpgradeVersion()); } finally { readUnlock(); } @@ -508,11 +490,10 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, * Causes heartbeat and lease daemons to stop; waits briefly for * them to finish, but a short timeout returns control back to caller. */ - public void close() { + void close() { fsRunning = false; try { if (blockManager != null) blockManager.close(); - if (hbthread != null) hbthread.interrupt(); if (smmthread != null) smmthread.interrupt(); if (dtSecretManager != null) dtSecretManager.stopThreads(); if (nnrmthread != null) nnrmthread.interrupt(); @@ -554,14 +535,12 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, long totalInodes = this.dir.totalInodes(); long totalBlocks = this.getBlocksTotal(); - - ArrayList live = new ArrayList(); - ArrayList dead = new ArrayList(); - this.DFSNodesStatus(live, dead); - - String str = totalInodes + " files and directories, " + totalBlocks - + " blocks = " + (totalInodes + totalBlocks) + " total"; - out.println(str); + out.println(totalInodes + " files and directories, " + totalBlocks + + " blocks = " + (totalInodes + totalBlocks) + " total"); + + final List live = new ArrayList(); + final List dead = new ArrayList(); + blockManager.getDatanodeManager().fetchDatanodes(live, dead, false); out.println("Live Datanodes: "+live.size()); out.println("Dead Datanodes: "+dead.size()); blockManager.metaSave(out); @@ -589,30 +568,6 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, return accessTimePrecision > 0; } - ///////////////////////////////////////////////////////// - // - // These methods are called by secondary namenodes - // - ///////////////////////////////////////////////////////// - /** - * return a list of blocks & their locations on datanode whose - * total size is size - * - * @param datanode on which blocks are located - * @param size total size of blocks - */ - BlocksWithLocations getBlocks(DatanodeID datanode, long size) - throws IOException { - readLock(); - try { - checkSuperuserPrivilege(); - return blockManager.getBlocksWithLocations(datanode, size); - } finally { - readUnlock(); - } - } - - ///////////////////////////////////////////////////////// // // These methods are called by HadoopFS clients @@ -622,7 +577,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, * Set permissions for an existing file. * @throws IOException */ - public void setPermission(String src, FsPermission permission) + void setPermission(String src, FsPermission permission) throws AccessControlException, FileNotFoundException, SafeModeException, UnresolvedLinkException, IOException { HdfsFileStatus resultingStat = null; @@ -651,7 +606,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, * Set owner for an existing file. * @throws IOException */ - public void setOwner(String src, String username, String group) + void setOwner(String src, String username, String group) throws AccessControlException, FileNotFoundException, SafeModeException, UnresolvedLinkException, IOException { HdfsFileStatus resultingStat = null; @@ -770,7 +725,9 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, } dir.setTimes(src, inode, -1, now, false); } - return getBlockLocationsInternal(inode, offset, length, needBlockToken); + return blockManager.createLocatedBlocks(inode.getBlocks(), + inode.computeFileSize(false), inode.isUnderConstruction(), + offset, length, needBlockToken); } finally { if (attempt == 0) { readUnlock(); @@ -781,50 +738,6 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, } return null; // can never reach here } - - LocatedBlocks getBlockLocationsInternal(INodeFile inode, - long offset, long length, boolean needBlockToken) - throws IOException { - assert hasReadOrWriteLock(); - final BlockInfo[] blocks = inode.getBlocks(); - if (LOG.isDebugEnabled()) { - LOG.debug("blocks = " + java.util.Arrays.asList(blocks)); - } - if (blocks == null) { - return null; - } - - if (blocks.length == 0) { - return new LocatedBlocks(0, inode.isUnderConstruction(), - Collections.emptyList(), null, false); - } else { - final long n = inode.computeFileSize(false); - final List locatedblocks = blockManager.getBlockLocations( - blocks, offset, length, Integer.MAX_VALUE); - final BlockInfo last = inode.getLastBlock(); - if (LOG.isDebugEnabled()) { - LOG.debug("last = " + last); - } - - LocatedBlock lastBlock = last.isComplete() ? blockManager - .getBlockLocation(last, n - last.getNumBytes()) : blockManager - .getBlockLocation(last, n); - - if (blockManager.isBlockTokenEnabled() && needBlockToken) { - blockManager.setBlockTokens(locatedblocks); - blockManager.setBlockToken(lastBlock); - } - return new LocatedBlocks(n, inode.isUnderConstruction(), locatedblocks, - lastBlock, last.isComplete()); - } - } - - /** Create a LocatedBlock. */ - public LocatedBlock createLocatedBlock(final Block b, final DatanodeInfo[] locations, - final long offset, final boolean corrupt) throws IOException { - return new LocatedBlock(getExtendedBlock(b), locations, offset, corrupt); - } - /** * Moves all the blocks from srcs and appends them to trg @@ -834,7 +747,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, * @param srcs * @throws IOException */ - public void concat(String target, String [] srcs) + void concat(String target, String [] srcs) throws IOException, UnresolvedLinkException { if(FSNamesystem.LOG.isDebugEnabled()) { FSNamesystem.LOG.debug("concat " + Arrays.toString(srcs) + @@ -882,7 +795,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, } /** See {@link #concat(String, String[])} */ - public void concatInternal(String target, String [] srcs) + private void concatInternal(String target, String [] srcs) throws IOException, UnresolvedLinkException { assert hasWriteLock(); @@ -986,7 +899,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, * The access time is precise upto an hour. The transaction, if needed, is * written to the edits log but is not flushed. */ - public void setTimes(String src, long mtime, long atime) + void setTimes(String src, long mtime, long atime) throws IOException, UnresolvedLinkException { if (!isAccessTimeSupported() && atime != -1) { throw new IOException("Access time for hdfs is not configured. " + @@ -1018,7 +931,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, /** * Create a symbolic link. */ - public void createSymlink(String target, String link, + void createSymlink(String target, String link, PermissionStatus dirPerms, boolean createParent) throws IOException, UnresolvedLinkException { HdfsFileStatus resultingStat = null; @@ -1086,60 +999,37 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, * @return true if successful; * false if file does not exist or is a directory */ - public boolean setReplication(String src, short replication) - throws IOException, UnresolvedLinkException { - boolean status = false; + boolean setReplication(final String src, final short replication + ) throws IOException { + blockManager.verifyReplication(src, replication, null); + + final boolean isFile; writeLock(); try { if (isInSafeMode()) { throw new SafeModeException("Cannot set replication for " + src, safeMode); } - status = setReplicationInternal(src, replication); + if (isPermissionEnabled) { + checkPathAccess(src, FsAction.WRITE); + } + + final short[] oldReplication = new short[1]; + final Block[] blocks = dir.setReplication(src, replication, oldReplication); + isFile = blocks != null; + if (isFile) { + blockManager.setReplication(oldReplication[0], replication, src, blocks); + } } finally { writeUnlock(); } + getEditLog().logSync(); - if (status && auditLog.isInfoEnabled() && isExternalInvocation()) { + if (isFile && auditLog.isInfoEnabled() && isExternalInvocation()) { logAuditEvent(UserGroupInformation.getCurrentUser(), Server.getRemoteIp(), "setReplication", src, null, null); } - return status; - } - - private boolean setReplicationInternal(String src, - short replication) throws AccessControlException, QuotaExceededException, - SafeModeException, UnresolvedLinkException, IOException { - assert hasWriteLock(); - blockManager.verifyReplication(src, replication, null); - if (isPermissionEnabled) { - checkPathAccess(src, FsAction.WRITE); - } - - int[] oldReplication = new int[1]; - Block[] fileBlocks; - fileBlocks = dir.setReplication(src, replication, oldReplication); - if (fileBlocks == null) // file not found or is a directory - return false; - int oldRepl = oldReplication[0]; - if (oldRepl == replication) // the same replication - return true; - - // update needReplication priority queues - for(int idx = 0; idx < fileBlocks.length; idx++) - blockManager.updateNeededReplications(fileBlocks[idx], 0, replication-oldRepl); - - if (oldRepl > replication) { - // old replication > the new one; need to remove copies - LOG.info("Reducing replication for file " + src - + ". New replication is " + replication); - for(int idx = 0; idx < fileBlocks.length; idx++) - blockManager.processOverReplicatedBlock(fileBlocks[idx], replication, null, null); - } else { // replication factor is increased - LOG.info("Increasing replication for file " + src - + ". New replication is " + replication); - } - return true; + return isFile; } long getPreferredBlockSize(String filename) @@ -1313,9 +1203,8 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, LocatedBlock lb = blockManager.convertLastBlockToUnderConstruction(cons); - if (lb != null && blockManager.isBlockTokenEnabled()) { - lb.setBlockToken(blockManager.getBlockTokenSecretManager().generateToken(lb.getBlock(), - EnumSet.of(BlockTokenSecretManager.AccessMode.WRITE))); + if (lb != null) { + blockManager.setBlockToken(lb, AccessMode.WRITE); } return lb; } else { @@ -1482,7 +1371,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, try { lb = startFileInternal(src, null, holder, clientMachine, EnumSet.of(CreateFlag.APPEND), - false, (short)blockManager.maxReplication, (long)0); + false, blockManager.maxReplication, (long)0); } finally { writeUnlock(); } @@ -1522,7 +1411,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, * are replicated. Will return an empty 2-elt array if we want the * client to "try again later". */ - public LocatedBlock getAdditionalBlock(String src, + LocatedBlock getAdditionalBlock(String src, String clientName, ExtendedBlock previous, HashMap excludedNodes @@ -1603,10 +1492,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, // Create next block LocatedBlock b = new LocatedBlock(getExtendedBlock(newBlock), targets, fileLength); - if (blockManager.isBlockTokenEnabled()) { - b.setBlockToken(blockManager.getBlockTokenSecretManager().generateToken(b.getBlock(), - EnumSet.of(BlockTokenSecretManager.AccessMode.WRITE))); - } + blockManager.setBlockToken(b, BlockTokenSecretManager.AccessMode.WRITE); return b; } @@ -1652,17 +1538,14 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, ).chooseTarget(src, numAdditionalNodes, clientnode, chosen, true, excludes, preferredblocksize); final LocatedBlock lb = new LocatedBlock(blk, targets); - if (blockManager.isBlockTokenEnabled()) { - lb.setBlockToken(blockManager.getBlockTokenSecretManager().generateToken(lb.getBlock(), - EnumSet.of(BlockTokenSecretManager.AccessMode.COPY))); - } + blockManager.setBlockToken(lb, AccessMode.COPY); return lb; } /** * The client would like to let go of the given block */ - public boolean abandonBlock(ExtendedBlock b, String src, String holder) + boolean abandonBlock(ExtendedBlock b, String src, String holder) throws LeaseExpiredException, FileNotFoundException, UnresolvedLinkException, IOException { writeLock(); @@ -1731,7 +1614,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, * (e.g if not all blocks have reached minimum replication yet) * @throws IOException on error (eg lease mismatch, file not open, file deleted) */ - public boolean completeFile(String src, String holder, ExtendedBlock last) + boolean completeFile(String src, String holder, ExtendedBlock last) throws SafeModeException, UnresolvedLinkException, IOException { checkBlock(last); boolean success = false; @@ -1847,23 +1730,6 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, } } - - /** - * Mark the block belonging to datanode as corrupt - * @param blk Block to be marked as corrupt - * @param dn Datanode which holds the corrupt replica - */ - public void markBlockAsCorrupt(ExtendedBlock blk, DatanodeInfo dn) - throws IOException { - writeLock(); - try { - blockManager.findAndMarkBlockAsCorrupt(blk.getLocalBlock(), dn); - } finally { - writeUnlock(); - } - } - - //////////////////////////////////////////////////////////////// // Here's how to handle block-copy failure during client write: // -- As usual, the client's write should result in a streaming @@ -1988,7 +1854,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, * @see ClientProtocol#delete(String, boolean) for detailed descriptoin and * description of exceptions */ - public boolean delete(String src, boolean recursive) + boolean delete(String src, boolean recursive) throws AccessControlException, SafeModeException, UnresolvedLinkException, IOException { if (NameNode.stateChangeLog.isDebugEnabled()) { @@ -2118,7 +1984,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, /** * Create all the necessary directories */ - public boolean mkdirs(String src, PermissionStatus permissions, + boolean mkdirs(String src, PermissionStatus permissions, boolean createParent) throws IOException, UnresolvedLinkException { boolean status = false; if(NameNode.stateChangeLog.isDebugEnabled()) { @@ -2374,7 +2240,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, return false; } - Lease reassignLease(Lease lease, String src, String newHolder, + private Lease reassignLease(Lease lease, String src, String newHolder, INodeFileUnderConstruction pendingFile) throws IOException { assert hasWriteLock(); if(newHolder == null) @@ -2390,6 +2256,22 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, return leaseManager.reassignLease(lease, src, newHolder); } + /** Update disk space consumed. */ + public void updateDiskSpaceConsumed(final INodeFileUnderConstruction fileINode, + final Block commitBlock) throws IOException { + assert hasWriteLock(); + + // Adjust disk space consumption if required + final long diff = fileINode.getPreferredBlockSize() - commitBlock.getNumBytes(); + if (diff > 0) { + try { + String path = leaseManager.findPath(fileINode); + dir.updateSpaceConsumed(path, 0, -diff * fileINode.getReplication()); + } catch (IOException e) { + LOG.warn("Unexpected exception while updating disk space.", e); + } + } + } private void finalizeINodeFileUnderConstruction(String src, INodeFileUnderConstruction pendingFile) @@ -2536,7 +2418,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, * @throws UnresolvedLinkException if symbolic link is encountered * @throws IOException if other I/O error occurred */ - public DirectoryListing getListing(String src, byte[] startAfter, + DirectoryListing getListing(String src, byte[] startAfter, boolean needLocation) throws AccessControlException, UnresolvedLinkException, IOException { DirectoryListing dl; @@ -2589,8 +2471,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, * * @see org.apache.hadoop.hdfs.server.datanode.DataNode */ - public void registerDatanode(DatanodeRegistration nodeReg) - throws IOException { + void registerDatanode(DatanodeRegistration nodeReg) throws IOException { writeLock(); try { getBlockManager().getDatanodeManager().registerDatanode(nodeReg); @@ -2606,7 +2487,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, * @see #registerDatanode(DatanodeRegistration) * @return registration ID */ - public String getRegistrationID() { + String getRegistrationID() { return Storage.getRegistrationID(dir.fsImage.getStorage()); } @@ -2621,13 +2502,14 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, * @return an array of datanode commands * @throws IOException */ - public DatanodeCommand[] handleHeartbeat(DatanodeRegistration nodeReg, + DatanodeCommand[] handleHeartbeat(DatanodeRegistration nodeReg, long capacity, long dfsUsed, long remaining, long blockPoolUsed, int xceiverCount, int xmitsInProgress, int failedVolumes) throws IOException { readLock(); try { - final int maxTransfer = blockManager.maxReplicationStreams - xmitsInProgress; + final int maxTransfer = blockManager.getMaxReplicationStreams() + - xmitsInProgress; DatanodeCommand[] cmds = blockManager.getDatanodeManager().handleHeartbeat( nodeReg, blockPoolId, capacity, dfsUsed, remaining, blockPoolUsed, xceiverCount, maxTransfer, failedVolumes); @@ -2636,7 +2518,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, } //check distributed upgrade - DatanodeCommand cmd = getDistributedUpgradeCommand(); + DatanodeCommand cmd = upgradeManager.getBroadcastCommand(); if (cmd != null) { return new DatanodeCommand[] {cmd}; } @@ -2646,45 +2528,6 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, } } - public void addKeyUpdateCommand(final List cmds, - final DatanodeDescriptor nodeinfo) { - // check access key update - if (blockManager.isBlockTokenEnabled() && nodeinfo.needKeyUpdate) { - cmds.add(new KeyUpdateCommand(blockManager.getBlockTokenSecretManager().exportKeys())); - nodeinfo.needKeyUpdate = false; - } - } - - public void updateStats(DatanodeDescriptor node, boolean isAdded) { - // - // The statistics are protected by the heartbeat lock - // For decommissioning/decommissioned nodes, only used capacity - // is counted. - // - assert(Thread.holdsLock(heartbeats)); - if (isAdded) { - capacityUsed += node.getDfsUsed(); - blockPoolUsed += node.getBlockPoolUsed(); - totalLoad += node.getXceiverCount(); - if (!(node.isDecommissionInProgress() || node.isDecommissioned())) { - capacityTotal += node.getCapacity(); - capacityRemaining += node.getRemaining(); - } else { - capacityTotal += node.getDfsUsed(); - } - } else { - capacityUsed -= node.getDfsUsed(); - blockPoolUsed -= node.getBlockPoolUsed(); - totalLoad -= node.getXceiverCount(); - if (!(node.isDecommissionInProgress() || node.isDecommissioned())) { - capacityTotal -= node.getCapacity(); - capacityRemaining -= node.getRemaining(); - } else { - capacityTotal -= node.getDfsUsed(); - } - } - } - /** * Returns whether or not there were available resources at the last check of * resources. @@ -2735,314 +2578,14 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, } } } - - - /** - * Periodically calls heartbeatCheck() and updateBlockKey() - */ - class HeartbeatMonitor implements Runnable { - private long lastHeartbeatCheck; - private long lastBlockKeyUpdate; - /** - */ - public void run() { - while (fsRunning) { - try { - long now = now(); - if (lastHeartbeatCheck + heartbeatRecheckInterval < now) { - heartbeatCheck(); - lastHeartbeatCheck = now; - } - if (blockManager.isBlockTokenEnabled() - && (lastBlockKeyUpdate + blockManager.getBlockKeyUpdateInterval() < now)) { - blockManager.updateBlockKey(); - lastBlockKeyUpdate = now; - } - } catch (Exception e) { - FSNamesystem.LOG.error("Exception while checking heartbeat", e); - } - try { - Thread.sleep(5000); // 5 seconds - } catch (InterruptedException ie) { - } - } - } - } - - - public void setNodeReplicationLimit(int limit) { - blockManager.maxReplicationStreams = limit; - } - - /** - * Remove a datanode descriptor. - * @param nodeID datanode ID. - * @throws UnregisteredNodeException - */ - public void removeDatanode(final DatanodeID nodeID - ) throws UnregisteredNodeException { - writeLock(); - try { - DatanodeDescriptor nodeInfo = getBlockManager().getDatanodeManager( - ).getDatanode(nodeID); - if (nodeInfo != null) { - removeDatanode(nodeInfo); - } else { - NameNode.stateChangeLog.warn("BLOCK* NameSystem.removeDatanode: " - + nodeID.getName() + " does not exist"); - } - } finally { - writeUnlock(); - } - } - /** - * Remove a datanode descriptor. - * @param nodeInfo datanode descriptor. - */ - public void removeDatanode(DatanodeDescriptor nodeInfo) { - assert hasWriteLock(); - synchronized (heartbeats) { - if (nodeInfo.isAlive) { - updateStats(nodeInfo, false); - heartbeats.remove(nodeInfo); - nodeInfo.isAlive = false; - } - } - - blockManager.removeDatanode(nodeInfo); - - checkSafeMode(); - } - FSImage getFSImage() { return dir.fsImage; } FSEditLog getEditLog() { return getFSImage().getEditLog(); - } - - /** - * Check if there are any expired heartbeats, and if so, - * whether any blocks have to be re-replicated. - * While removing dead datanodes, make sure that only one datanode is marked - * dead at a time within the synchronized section. Otherwise, a cascading - * effect causes more datanodes to be declared dead. - */ - void heartbeatCheck() { - final DatanodeManager datanodeManager = getBlockManager().getDatanodeManager(); - // It's OK to check safe mode w/o taking the lock here, we re-check - // for safe mode after taking the lock before removing a datanode. - if (isInSafeMode()) { - return; - } - boolean allAlive = false; - while (!allAlive) { - boolean foundDead = false; - DatanodeID nodeID = null; - - // locate the first dead node. - synchronized(heartbeats) { - for (Iterator it = heartbeats.iterator(); - it.hasNext();) { - DatanodeDescriptor nodeInfo = it.next(); - if (datanodeManager.isDatanodeDead(nodeInfo)) { - expiredHeartbeats.incr(); - foundDead = true; - nodeID = nodeInfo; - break; - } - } - } - - // acquire the fsnamesystem lock, and then remove the dead node. - if (foundDead) { - writeLock(); - if (isInSafeMode()) { - return; - } - try { - datanodeManager.removeDeadDatanode(nodeID); - } finally { - writeUnlock(); - } - } - allAlive = !foundDead; - } - } - - /** - * The given node is reporting all its blocks. Use this info to - * update the (machine-->blocklist) and (block-->machinelist) tables. - */ - public void processReport(DatanodeID nodeID, String poolId, - BlockListAsLongs newReport) throws IOException { - long startTime, endTime; - - writeLock(); - startTime = now(); //after acquiring write lock - try { - final DatanodeDescriptor node = blockManager.getDatanodeManager( - ).getDatanode(nodeID); - if (node == null || !node.isAlive) { - throw new IOException("ProcessReport from dead or unregistered node: " - + nodeID.getName()); - } - // To minimize startup time, we discard any second (or later) block reports - // that we receive while still in startup phase. - if (isInStartupSafeMode() && node.numBlocks() > 0) { - NameNode.stateChangeLog.info("BLOCK* NameSystem.processReport: " - + "discarded non-initial block report from " + nodeID.getName() - + " because namenode still in startup phase"); - return; - } - - blockManager.processReport(node, newReport); - } finally { - endTime = now(); - writeUnlock(); - } - - // Log the block report processing stats from Namenode perspective - NameNode.getNameNodeMetrics().addBlockReport((int) (endTime - startTime)); - NameNode.stateChangeLog.info("BLOCK* NameSystem.processReport: from " - + nodeID.getName() + ", blocks: " + newReport.getNumberOfBlocks() - + ", processing time: " + (endTime - startTime) + " msecs"); - } - - /** - * We want "replication" replicates for the block, but we now have too many. - * In this method, copy enough nodes from 'srcNodes' into 'dstNodes' such that: - * - * srcNodes.size() - dstNodes.size() == replication - * - * We pick node that make sure that replicas are spread across racks and - * also try hard to pick one with least free space. - * The algorithm is first to pick a node with least free space from nodes - * that are on a rack holding more than one replicas of the block. - * So removing such a replica won't remove a rack. - * If no such a node is available, - * then pick a node with least free space - */ - public void chooseExcessReplicates(Collection nonExcess, - Block b, short replication, - DatanodeDescriptor addedNode, - DatanodeDescriptor delNodeHint, - BlockPlacementPolicy replicator) { - assert hasWriteLock(); - // first form a rack to datanodes map and - INodeFile inode = blockManager.getINode(b); - HashMap> rackMap = - new HashMap>(); - for (Iterator iter = nonExcess.iterator(); - iter.hasNext();) { - DatanodeDescriptor node = iter.next(); - String rackName = node.getNetworkLocation(); - ArrayList datanodeList = rackMap.get(rackName); - if(datanodeList==null) { - datanodeList = new ArrayList(); - } - datanodeList.add(node); - rackMap.put(rackName, datanodeList); - } - - // split nodes into two sets - // priSet contains nodes on rack with more than one replica - // remains contains the remaining nodes - ArrayList priSet = new ArrayList(); - ArrayList remains = new ArrayList(); - for( Iterator>> iter = - rackMap.entrySet().iterator(); iter.hasNext(); ) { - Entry> rackEntry = iter.next(); - ArrayList datanodeList = rackEntry.getValue(); - if( datanodeList.size() == 1 ) { - remains.add(datanodeList.get(0)); - } else { - priSet.addAll(datanodeList); - } - } - - // pick one node to delete that favors the delete hint - // otherwise pick one with least space from priSet if it is not empty - // otherwise one node with least space from remains - boolean firstOne = true; - while (nonExcess.size() - replication > 0) { - DatanodeInfo cur = null; - - // check if we can del delNodeHint - if (firstOne && delNodeHint !=null && nonExcess.contains(delNodeHint) && - (priSet.contains(delNodeHint) || (addedNode != null && !priSet.contains(addedNode))) ) { - cur = delNodeHint; - } else { // regular excessive replica removal - cur = replicator.chooseReplicaToDelete(inode, b, replication, priSet, remains); - } - - firstOne = false; - // adjust rackmap, priSet, and remains - String rack = cur.getNetworkLocation(); - ArrayList datanodes = rackMap.get(rack); - datanodes.remove(cur); - if(datanodes.isEmpty()) { - rackMap.remove(rack); - } - if( priSet.remove(cur) ) { - if (datanodes.size() == 1) { - priSet.remove(datanodes.get(0)); - remains.add(datanodes.get(0)); - } - } else { - remains.remove(cur); - } - - nonExcess.remove(cur); - blockManager.addToExcessReplicate(cur, b); - - // - // The 'excessblocks' tracks blocks until we get confirmation - // that the datanode has deleted them; the only way we remove them - // is when we get a "removeBlock" message. - // - // The 'invalidate' list is used to inform the datanode the block - // should be deleted. Items are removed from the invalidate list - // upon giving instructions to the namenode. - // - blockManager.addToInvalidates(b, cur); - NameNode.stateChangeLog.info("BLOCK* NameSystem.chooseExcessReplicates: " - +"("+cur.getName()+", "+b+") is added to recentInvalidateSets"); - } - } - - - /** - * The given node is reporting that it received a certain block. - */ - public void blockReceived(DatanodeID nodeID, - String poolId, - Block block, - String delHint - ) throws IOException { - writeLock(); - try { - final DatanodeDescriptor node = blockManager.getDatanodeManager( - ).getDatanode(nodeID); - if (node == null || !node.isAlive) { - NameNode.stateChangeLog.warn("BLOCK* NameSystem.blockReceived: " + block - + " is received from dead or unregistered node " + nodeID.getName()); - throw new IOException( - "Got blockReceived message from unregistered or dead node " + block); - } - - if (NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug("BLOCK* NameSystem.blockReceived: " - +block+" is received from " + nodeID.getName()); - } - - blockManager.addBlock(node, block, delHint); - } finally { - writeUnlock(); - } - } + } private void checkBlock(ExtendedBlock block) throws IOException { if (block != null && !this.blockPoolId.equals(block.getBlockPoolId())) { @@ -3057,15 +2600,18 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, return blockManager.getMissingBlocksCount(); } + /** Increment expired heartbeat counter. */ + public void incrExpiredHeartbeats() { + expiredHeartbeats.incr(); + } + + /** @see ClientProtocol#getStats() */ long[] getStats() { - synchronized(heartbeats) { - return new long[] {this.capacityTotal, this.capacityUsed, - this.capacityRemaining, - getUnderReplicatedBlocks(), - getCorruptReplicaBlocks(), - getMissingBlocksCount(), - getBlockPoolUsedSpace()}; - } + final long[] stats = datanodeStatistics.getStats(); + stats[ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX] = getUnderReplicatedBlocks(); + stats[ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX] = getCorruptReplicaBlocks(); + stats[ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX] = getMissingBlocksCount(); + return stats; } /** @@ -3073,9 +2619,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, */ @Override // FSNamesystemMBean public long getCapacityTotal() { - synchronized(heartbeats) { - return capacityTotal; - } + return datanodeStatistics.getCapacityTotal(); } @Metric @@ -3088,9 +2632,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, */ @Override // FSNamesystemMBean public long getCapacityUsed() { - synchronized(heartbeats) { - return capacityUsed; - } + return datanodeStatistics.getCapacityUsed(); } @Metric @@ -3098,32 +2640,9 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, return DFSUtil.roundBytesToGB(getCapacityUsed()); } - /** - * Total used space by data nodes as percentage of total capacity - */ - public float getCapacityUsedPercent() { - synchronized(heartbeats){ - return DFSUtil.getPercentUsed(capacityUsed, capacityTotal); - } - } - /** - * Total used space by data nodes for non DFS purposes such - * as storing temporary files on the local file system - */ - public long getCapacityUsedNonDFS() { - long nonDFSUsed = 0; - synchronized(heartbeats){ - nonDFSUsed = capacityTotal - capacityRemaining - capacityUsed; - } - return nonDFSUsed < 0 ? 0 : nonDFSUsed; - } - /** - * Total non-used raw bytes. - */ + @Override public long getCapacityRemaining() { - synchronized(heartbeats) { - return capacityRemaining; - } + return datanodeStatistics.getCapacityRemaining(); } @Metric @@ -3131,23 +2650,13 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, return DFSUtil.roundBytesToGB(getCapacityRemaining()); } - /** - * Total remaining space by data nodes as percentage of total capacity - */ - public float getCapacityRemainingPercent() { - synchronized(heartbeats){ - return DFSUtil.getPercentRemaining(capacityRemaining, capacityTotal); - } - } /** * Total number of connections. */ @Override // FSNamesystemMBean @Metric public int getTotalLoad() { - synchronized (heartbeats) { - return this.totalLoad; - } + return datanodeStatistics.getXceiverCount(); } int getNumberOfDatanodes(DatanodeReportType type) { @@ -3225,63 +2734,10 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, } } - /** - */ - public void DFSNodesStatus(ArrayList live, - ArrayList dead) { - readLock(); - try { - getBlockManager().getDatanodeManager().fetchDatanodess(live, dead); - } finally { - readUnlock(); - } - } - - public Date getStartTime() { + Date getStartTime() { return new Date(systemStart); } - short getMaxReplication() { return (short)blockManager.maxReplication; } - short getMinReplication() { return (short)blockManager.minReplication; } - short getDefaultReplication() { return (short)blockManager.defaultReplication; } - - /** - * Clamp the specified replication between the minimum and maximum - * replication levels for this namesystem. - */ - short adjustReplication(short replication) { - short minReplication = getMinReplication(); - if (replication < minReplication) { - replication = minReplication; - } - short maxReplication = getMaxReplication(); - if (replication > maxReplication) { - replication = maxReplication; - } - return replication; - } - - - /** - * Rereads the config to get hosts and exclude list file names. - * Rereads the files to update the hosts and exclude lists. It - * checks if any of the hosts have changed states: - * 1. Added to hosts --> no further work needed here. - * 2. Removed from hosts --> mark AdminState as decommissioned. - * 3. Added to exclude --> start decommission. - * 4. Removed from exclude --> stop decommission. - */ - public void refreshNodes(Configuration conf) throws IOException { - checkSuperuserPrivilege(); - getBlockManager().getDatanodeManager().refreshHostsReader(conf); - writeLock(); - try { - getBlockManager().getDatanodeManager().refreshDatanodes(); - } finally { - writeUnlock(); - } - } - void finalizeUpgrade() throws IOException { checkSuperuserPrivilege(); getFSImage().finalizeUpgrade(); @@ -3351,16 +2807,17 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, * @param conf configuration */ SafeModeInfo(Configuration conf) { - this.threshold = conf.getFloat(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY, DFSConfigKeys.DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_DEFAULT); + this.threshold = conf.getFloat(DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY, + DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_DEFAULT); this.datanodeThreshold = conf.getInt( - DFSConfigKeys.DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY, - DFSConfigKeys.DFS_NAMENODE_SAFEMODE_MIN_DATANODES_DEFAULT); - this.extension = conf.getInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, 0); - this.safeReplication = conf.getInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY, - DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_DEFAULT); + DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY, + DFS_NAMENODE_SAFEMODE_MIN_DATANODES_DEFAULT); + this.extension = conf.getInt(DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, 0); + this.safeReplication = conf.getInt(DFS_NAMENODE_REPLICATION_MIN_KEY, + DFS_NAMENODE_REPLICATION_MIN_DEFAULT); // default to safe mode threshold (i.e., don't populate queues before leaving safe mode) this.replQueueThreshold = - conf.getFloat(DFSConfigKeys.DFS_NAMENODE_REPL_QUEUE_THRESHOLD_PCT_KEY, + conf.getFloat(DFS_NAMENODE_REPL_QUEUE_THRESHOLD_PCT_KEY, (float) threshold); this.blockTotal = 0; this.blockSafe = 0; @@ -3429,7 +2886,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, // verify whether a distributed upgrade needs to be started boolean needUpgrade = false; try { - needUpgrade = startDistributedUpgradeIfNeeded(); + needUpgrade = upgradeManager.startUpgrade(); } catch(IOException e) { FSNamesystem.LOG.error("IOException in startDistributedUpgradeIfNeeded", e); } @@ -3458,7 +2915,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, + nt.getNumOfRacks() + " racks and " + nt.getNumOfLeaves() + " datanodes"); NameNode.stateChangeLog.info("STATE* UnderReplicatedBlocks has " - +blockManager.neededReplications.size()+" blocks"); + + blockManager.numOfUnderReplicatedBlocks() + " blocks"); } /** @@ -3622,10 +3079,10 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, leaveMsg = "Safe mode will be turned off automatically"; } if(isManual()) { - if(getDistributedUpgradeState()) + if(upgradeManager.getUpgradeState()) return leaveMsg + " upon completion of " + "the distributed upgrade: upgrade progress = " + - getDistributedUpgradeStatus() + "%"; + upgradeManager.getUpgradeStatus() + "%"; leaveMsg = "Use \"hdfs dfsadmin -safemode leave\" to turn safe mode off"; } @@ -3757,8 +3214,9 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, } return isInSafeMode(); } - - private void checkSafeMode() { + + /** Check and trigger safe mode. */ + public void checkSafeMode() { // safeMode is volatile, and may be set to null at any time SafeModeInfo safeMode = this.safeMode; if (safeMode != null) { @@ -3826,7 +3284,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, /** * Set the total number of blocks in the system. */ - void setBlockTotal() { + private void setBlockTotal() { // safeMode is volatile, and may be set to null at any time SafeModeInfo safeMode = this.safeMode; if (safeMode == null) @@ -3847,7 +3305,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, * Get the total number of COMPLETE blocks in the system. * For safe mode only complete blocks are counted. */ - long getCompleteBlocksTotal() { + private long getCompleteBlocksTotal() { // Calculate number of blocks under construction long numUCBlocks = 0; readLock(); @@ -3918,7 +3376,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, NameNode.stateChangeLog.info("STATE* Safe mode is already OFF."); return; } - if(getDistributedUpgradeState()) + if(upgradeManager.getUpgradeState()) throw new SafeModeException("Distributed upgrade is in progress", safeMode); safeMode.leave(checkForUpgrades); @@ -3955,10 +3413,6 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, writeUnlock(); } } - - public RemoteEditLogManifest getEditLogManifest(long sinceTxId) throws IOException { - return getEditLog().getEditLogManifest(sinceTxId); - } NamenodeCommand startCheckpoint( NamenodeRegistration bnReg, // backup node @@ -4011,26 +3465,6 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, return upgradeManager.processUpgradeCommand(comm); } - int getDistributedUpgradeVersion() { - return upgradeManager.getUpgradeVersion(); - } - - UpgradeCommand getDistributedUpgradeCommand() throws IOException { - return upgradeManager.getBroadcastCommand(); - } - - boolean getDistributedUpgradeState() { - return upgradeManager.getUpgradeState(); - } - - short getDistributedUpgradeStatus() { - return upgradeManager.getUpgradeStatus(); - } - - boolean startDistributedUpgradeIfNeeded() throws IOException { - return upgradeManager.startUpgrade(); - } - PermissionStatus createFsOwnerPermissions(FsPermission permission) { return new PermissionStatus(fsOwner.getShortUserName(), supergroup, permission); } @@ -4060,7 +3494,8 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, return checkPermission(path, false, null, null, null, null); } - private void checkSuperuserPrivilege() throws AccessControlException { + /** Check if the user has superuser privilege. */ + public void checkSuperuserPrivilege() throws AccessControlException { if (isPermissionEnabled) { FSPermissionChecker.checkSuperuserPrivilege(fsOwner, supergroup); } @@ -4131,11 +3566,6 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, return blockManager.getUnderReplicatedBlocksCount(); } - /** Return number of under-replicated but not missing blocks */ - public long getUnderReplicatedNotMissingBlocks() { - return blockManager.getUnderReplicatedNotMissingBlocks(); - } - /** Returns number of blocks with corrupt replicas */ @Metric({"CorruptBlocks", "Number of blocks with corrupt replicas"}) public long getCorruptReplicaBlocks() { @@ -4173,7 +3603,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, * Register the FSNamesystem MBean using the name * "hadoop:service=NameNode,name=FSNamesystemState" */ - void registerMBean() { + private void registerMBean() { // We can only implement one MXBean interface, so we keep the old one. try { StandardMBean bean = new StandardMBean(this, FSNamesystemMBean.class); @@ -4188,7 +3618,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, /** * shutdown FSNamesystem */ - public void shutdown() { + void shutdown() { if (mbeanName != null) MBeans.unregister(mbeanName); } @@ -4207,14 +3637,14 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, /** * Sets the generation stamp for this filesystem */ - public void setGenerationStamp(long stamp) { + void setGenerationStamp(long stamp) { generationStamp.setStamp(stamp); } /** * Gets the generation stamp for this filesystem */ - public long getGenerationStamp() { + long getGenerationStamp() { return generationStamp.getStamp(); } @@ -4289,10 +3719,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, // get a new generation stamp and an access token block.setGenerationStamp(nextGenerationStamp()); locatedBlock = new LocatedBlock(block, new DatanodeInfo[0]); - if (blockManager.isBlockTokenEnabled()) { - locatedBlock.setBlockToken(blockManager.getBlockTokenSecretManager().generateToken( - block, EnumSet.of(BlockTokenSecretManager.AccessMode.WRITE))); - } + blockManager.setBlockToken(locatedBlock, AccessMode.WRITE); } finally { writeUnlock(); } @@ -4337,7 +3764,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, } /** @see updatePipeline(String, ExtendedBlock, ExtendedBlock, DatanodeID[]) */ - void updatePipelineInternal(String clientName, ExtendedBlock oldBlock, + private void updatePipelineInternal(String clientName, ExtendedBlock oldBlock, ExtendedBlock newBlock, DatanodeID[] newNodes) throws IOException { assert hasWriteLock(); @@ -4493,26 +3920,6 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, return blockManager.numCorruptReplicas(blk); } - /** - * Return a range of corrupt replica block ids. Up to numExpectedBlocks - * blocks starting at the next block after startingBlockId are returned - * (fewer if numExpectedBlocks blocks are unavailable). If startingBlockId - * is null, up to numExpectedBlocks blocks are returned from the beginning. - * If startingBlockId cannot be found, null is returned. - * - * @param numExpectedBlocks Number of block ids to return. - * 0 <= numExpectedBlocks <= 100 - * @param startingBlockId Block id from which to start. If null, start at - * beginning. - * @return Up to numExpectedBlocks blocks from startingBlockId if it exists - * - */ - long[] getCorruptReplicaBlockIds(int numExpectedBlocks, - Long startingBlockId) { - return blockManager.getCorruptReplicaBlockIds(numExpectedBlocks, - startingBlockId); - } - static class CorruptFileBlockInfo { String path; Block block; @@ -4552,7 +3959,8 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, if (startBlockAfter != null) { startBlockId = Block.filename2id(startBlockAfter); } - UnderReplicatedBlocks.BlockIterator blkIterator = blockManager.getCorruptReplicaBlockIterator(); + + final Iterator blkIterator = blockManager.getCorruptReplicaBlockIterator(); while (blkIterator.hasNext()) { Block blk = blkIterator.next(); INode inode = blockManager.getINode(blk); @@ -4574,42 +3982,18 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, } } - /** - * @return list of datanodes where decommissioning is in progress - */ - public ArrayList getDecommissioningNodes() { - readLock(); - try { - ArrayList decommissioningNodes = - new ArrayList(); - final List results = getBlockManager( - ).getDatanodeManager().getDatanodeListForReport(DatanodeReportType.LIVE); - for (Iterator it = results.iterator(); it.hasNext();) { - DatanodeDescriptor node = it.next(); - if (node.isDecommissionInProgress()) { - decommissioningNodes.add(node); - } - } - return decommissioningNodes; - } finally { - readUnlock(); - } - } - /** * Create delegation token secret manager */ private DelegationTokenSecretManager createDelegationTokenSecretManager( Configuration conf) { return new DelegationTokenSecretManager(conf.getLong( - DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_KEY, - DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT), - conf.getLong( - DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, - DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT), - conf.getLong( - DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, - DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT), + DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_KEY, + DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT), + conf.getLong(DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, + DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT), + conf.getLong(DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, + DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT), DELEGATION_TOKEN_REMOVER_SCAN_INTERVAL, this); } @@ -4617,7 +4001,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, * Returns the DelegationTokenSecretManager instance in the namesystem. * @return delegation token secret manager object */ - public DelegationTokenSecretManager getDelegationTokenSecretManager() { + DelegationTokenSecretManager getDelegationTokenSecretManager() { return dtSecretManager; } @@ -4626,7 +4010,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, * @return Token * @throws IOException */ - public Token getDelegationToken(Text renewer) + Token getDelegationToken(Text renewer) throws IOException { Token token; writeLock(); @@ -4670,7 +4054,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, * @throws InvalidToken * @throws IOException */ - public long renewDelegationToken(Token token) + long renewDelegationToken(Token token) throws InvalidToken, IOException { long expiryTime; writeLock(); @@ -4701,7 +4085,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, * @param token * @throws IOException */ - public void cancelDelegationToken(Token token) + void cancelDelegationToken(Token token) throws IOException { writeLock(); try { @@ -4854,31 +4238,27 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, @Override // NameNodeMXBean public long getNonDfsUsedSpace() { - return getCapacityUsedNonDFS(); + return datanodeStatistics.getCapacityUsedNonDFS(); } @Override // NameNodeMXBean public float getPercentUsed() { - return getCapacityUsedPercent(); + return datanodeStatistics.getCapacityUsedPercent(); } @Override // NameNodeMXBean public long getBlockPoolUsedSpace() { - synchronized(heartbeats) { - return blockPoolUsed; - } + return datanodeStatistics.getBlockPoolUsed(); } @Override // NameNodeMXBean public float getPercentBlockPoolUsed() { - synchronized(heartbeats) { - return DFSUtil.getPercentUsed(blockPoolUsed, capacityTotal); - } + return datanodeStatistics.getPercentBlockPoolUsed(); } @Override // NameNodeMXBean public float getPercentRemaining() { - return getCapacityRemainingPercent(); + return datanodeStatistics.getCapacityRemainingPercent(); } @Override // NameNodeMXBean @@ -4910,13 +4290,9 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, public String getLiveNodes() { final Map> info = new HashMap>(); - final ArrayList liveNodeList = - new ArrayList(); - final ArrayList deadNodeList = - new ArrayList(); - DFSNodesStatus(liveNodeList, deadNodeList); - removeDecomNodeFromList(liveNodeList); - for (DatanodeDescriptor node : liveNodeList) { + final List live = new ArrayList(); + blockManager.getDatanodeManager().fetchDatanodes(live, null, true); + for (DatanodeDescriptor node : live) { final Map innerinfo = new HashMap(); innerinfo.put("lastContact", getLastContact(node)); innerinfo.put("usedSpace", getDfsUsed(node)); @@ -4934,14 +4310,9 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, public String getDeadNodes() { final Map> info = new HashMap>(); - final ArrayList liveNodeList = - new ArrayList(); - final ArrayList deadNodeList = - new ArrayList(); - // we need to call DFSNodeStatus to filter out the dead data nodes - DFSNodesStatus(liveNodeList, deadNodeList); - removeDecomNodeFromList(deadNodeList); - for (DatanodeDescriptor node : deadNodeList) { + final List dead = new ArrayList(); + blockManager.getDatanodeManager().fetchDatanodes(null, dead, true); + for (DatanodeDescriptor node : dead) { final Map innerinfo = new HashMap(); innerinfo.put("lastContact", getLastContact(node)); innerinfo.put("decommissioned", node.isDecommissioned()); @@ -4958,8 +4329,8 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, public String getDecomNodes() { final Map> info = new HashMap>(); - final ArrayList decomNodeList = - this.getDecommissioningNodes(); + final List decomNodeList = blockManager.getDatanodeManager( + ).getDecommissioningNodes(); for (DatanodeDescriptor node : decomNodeList) { final Map innerinfo = new HashMap(); innerinfo.put("underReplicatedBlocks", node.decommissioningStatus @@ -4995,18 +4366,4 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, public BlockManager getBlockManager() { return blockManager; } - - void removeDecomNodeFromList(List nodeList) { - getBlockManager().getDatanodeManager().removeDecomNodeFromList(nodeList); - } - - /** - * Tell all datanodes to use a new, non-persistent bandwidth value for - * dfs.datanode.balance.bandwidthPerSec. - * @param bandwidth Blanacer bandwidth in bytes per second for all datanodes. - * @throws IOException - */ - public void setBalancerBandwidth(long bandwidth) throws IOException { - getBlockManager().getDatanodeManager().setBalancerBandwidth(bandwidth); - } } diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java index 7fc7edbdb24..b7587c0dd1c 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.io.PrintWriter; import java.net.URI; import java.net.URISyntaxException; +import java.net.URL; import javax.net.SocketFactory; import javax.servlet.ServletContext; @@ -36,11 +37,14 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.DatanodeID; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.server.common.HdfsConstants; +import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DatanodeJspHelper; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.ServletUtil; import org.znerd.xmlenc.XMLOutputter; /** Servlets for file checksum */ @@ -52,6 +56,32 @@ public class FileChecksumServlets { /** For java.io.Serializable */ private static final long serialVersionUID = 1L; + /** Create a redirection URL */ + private URL createRedirectURL(UserGroupInformation ugi, DatanodeID host, + HttpServletRequest request, NameNode nn) + throws IOException { + final String hostname = host instanceof DatanodeInfo + ? ((DatanodeInfo)host).getHostName() : host.getHost(); + final String scheme = request.getScheme(); + final int port = "https".equals(scheme) + ? (Integer)getServletContext().getAttribute("datanode.https.port") + : host.getInfoPort(); + final String encodedPath = ServletUtil.getRawPath(request, "/fileChecksum"); + + String dtParam = ""; + if (UserGroupInformation.isSecurityEnabled()) { + String tokenString = ugi.getTokens().iterator().next().encodeToUrlString(); + dtParam = JspHelper.getDelegationTokenUrlParam(tokenString); + } + String addr = NameNode.getHostPortString(nn.getNameNodeAddress()); + String addrParam = JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, addr); + + return new URL(scheme, hostname, port, + "/getFileChecksum" + encodedPath + '?' + + "ugi=" + ServletUtil.encodeQueryValue(ugi.getShortUserName()) + + dtParam + addrParam); + } + /** {@inheritDoc} */ public void doGet(HttpServletRequest request, HttpServletResponse response ) throws ServletException, IOException { @@ -62,12 +92,8 @@ public class FileChecksumServlets { context); final DatanodeID datanode = NamenodeJspHelper.getRandomDatanode(namenode); try { - final URI uri = createRedirectUri("/getFileChecksum", ugi, datanode, - request, namenode); - response.sendRedirect(uri.toURL().toString()); - } catch(URISyntaxException e) { - throw new ServletException(e); - //response.getWriter().println(e.toString()); + response.sendRedirect( + createRedirectURL(ugi, datanode, request, namenode).toString()); } catch (IOException e) { response.sendError(400, e.getMessage()); } @@ -84,7 +110,7 @@ public class FileChecksumServlets { public void doGet(HttpServletRequest request, HttpServletResponse response ) throws ServletException, IOException { final PrintWriter out = response.getWriter(); - final String filename = getFilename(request, response); + final String path = ServletUtil.getDecodedPath(request, "/getFileChecksum"); final XMLOutputter xml = new XMLOutputter(out, "UTF-8"); xml.declaration(); @@ -103,12 +129,12 @@ public class FileChecksumServlets { datanode, conf, getUGI(request, conf)); final ClientProtocol nnproxy = dfs.getNamenode(); final MD5MD5CRC32FileChecksum checksum = DFSClient.getFileChecksum( - filename, nnproxy, socketFactory, socketTimeout); + path, nnproxy, socketFactory, socketTimeout); MD5MD5CRC32FileChecksum.write(xml, checksum); } catch(IOException ioe) { - writeXml(ioe, filename, xml); + writeXml(ioe, path, xml); } catch (InterruptedException e) { - writeXml(e, filename, xml); + writeXml(e, path, xml); } xml.endDocument(); } diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FileDataServlet.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FileDataServlet.java index 9573caa6a62..54b0ec64abc 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FileDataServlet.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FileDataServlet.java @@ -18,8 +18,8 @@ package org.apache.hadoop.hdfs.server.namenode; import java.io.IOException; -import java.net.URI; import java.net.URISyntaxException; +import java.net.URL; import java.security.PrivilegedExceptionAction; import javax.servlet.http.HttpServletRequest; @@ -35,6 +35,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.ServletUtil; /** Redirect queries about the hosted filesystem to an appropriate datanode. * @see org.apache.hadoop.hdfs.HftpFileSystem @@ -44,22 +45,25 @@ public class FileDataServlet extends DfsServlet { /** For java.io.Serializable */ private static final long serialVersionUID = 1L; - /** Create a redirection URI */ - protected URI createUri(String parent, HdfsFileStatus i, UserGroupInformation ugi, - ClientProtocol nnproxy, HttpServletRequest request, String dt) - throws IOException, URISyntaxException { + /** Create a redirection URL */ + private URL createRedirectURL(String path, String encodedPath, HdfsFileStatus status, + UserGroupInformation ugi, ClientProtocol nnproxy, HttpServletRequest request, String dt) + throws IOException { String scheme = request.getScheme(); final LocatedBlocks blks = nnproxy.getBlockLocations( - i.getFullPath(new Path(parent)).toUri().getPath(), 0, 1); - final DatanodeID host = pickSrcDatanode(blks, i); + status.getFullPath(new Path(path)).toUri().getPath(), 0, 1); + final DatanodeID host = pickSrcDatanode(blks, status); final String hostname; if (host instanceof DatanodeInfo) { hostname = ((DatanodeInfo)host).getHostName(); } else { hostname = host.getHost(); } - - String dtParam=""; + final int port = "https".equals(scheme) + ? (Integer)getServletContext().getAttribute("datanode.https.port") + : host.getInfoPort(); + + String dtParam = ""; if (dt != null) { dtParam=JspHelper.getDelegationTokenUrlParam(dt); } @@ -70,12 +74,10 @@ public class FileDataServlet extends DfsServlet { String addr = NameNode.getHostPortString(nn.getNameNodeAddress()); String addrParam = JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, addr); - return new URI(scheme, null, hostname, - "https".equals(scheme) - ? (Integer)getServletContext().getAttribute("datanode.https.port") - : host.getInfoPort(), - "/streamFile" + i.getFullName(parent), - "ugi=" + ugi.getShortUserName() + dtParam + addrParam, null); + return new URL(scheme, hostname, port, + "/streamFile" + encodedPath + '?' + + "ugi=" + ServletUtil.encodeQueryValue(ugi.getShortUserName()) + + dtParam + addrParam); } /** Select a datanode to service this request. @@ -112,20 +114,15 @@ public class FileDataServlet extends DfsServlet { @Override public Void run() throws IOException { ClientProtocol nn = createNameNodeProxy(); - final String path = request.getPathInfo() != null ? request - .getPathInfo() : "/"; - + final String path = ServletUtil.getDecodedPath(request, "/data"); + final String encodedPath = ServletUtil.getRawPath(request, "/data"); String delegationToken = request .getParameter(JspHelper.DELEGATION_PARAMETER_NAME); HdfsFileStatus info = nn.getFileInfo(path); if (info != null && !info.isDir()) { - try { - response.sendRedirect(createUri(path, info, ugi, nn, request, - delegationToken).toURL().toString()); - } catch (URISyntaxException e) { - response.getWriter().println(e.toString()); - } + response.sendRedirect(createRedirectURL(path, encodedPath, + info, ugi, nn, request, delegationToken).toString()); } else if (info == null) { response.sendError(400, "File not found " + path); } else { diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java index 360a118fa27..991d7f5c5dd 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java @@ -23,14 +23,21 @@ import org.apache.commons.logging.LogFactory; import java.io.File; import java.io.IOException; import java.util.List; +import java.util.Comparator; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; -import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundEditLog; import org.apache.hadoop.hdfs.server.namenode.NNStorageRetentionManager.StoragePurger; +import org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation; +import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile; +import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.ComparisonChain; /** * Journal manager for the common case of edits files being written @@ -45,6 +52,15 @@ class FileJournalManager implements JournalManager { private final StorageDirectory sd; private int outputBufferCapacity = 512*1024; + private static final Pattern EDITS_REGEX = Pattern.compile( + NameNodeFile.EDITS.getName() + "_(\\d+)-(\\d+)"); + private static final Pattern EDITS_INPROGRESS_REGEX = Pattern.compile( + NameNodeFile.EDITS_INPROGRESS.getName() + "_(\\d+)"); + + @VisibleForTesting + StoragePurger purger + = new NNStorageRetentionManager.DeletionStoragePurger(); + public FileJournalManager(StorageDirectory sd) { this.sd = sd; } @@ -91,13 +107,13 @@ class FileJournalManager implements JournalManager { } @Override - public void purgeLogsOlderThan(long minTxIdToKeep, StoragePurger purger) + public void purgeLogsOlderThan(long minTxIdToKeep) throws IOException { File[] files = FileUtil.listFiles(sd.getCurrentDir()); - List editLogs = - FSImageTransactionalStorageInspector.matchEditLogs(files); - for (FoundEditLog log : editLogs) { - if (log.getStartTxId() < minTxIdToKeep && + List editLogs = + FileJournalManager.matchEditLogs(files); + for (EditLogFile log : editLogs) { + if (log.getFirstTxId() < minTxIdToKeep && log.getLastTxId() < minTxIdToKeep) { purger.purgeLog(log); } @@ -110,5 +126,167 @@ class FileJournalManager implements JournalManager { File f = NNStorage.getInProgressEditsFile(sd, segmentStartsAtTxId); return new EditLogFileInputStream(f); } + + /** + * Find all editlog segments starting at or above the given txid. + * @param fromTxId the txnid which to start looking + * @return a list of remote edit logs + * @throws IOException if edit logs cannot be listed. + */ + List getRemoteEditLogs(long firstTxId) throws IOException { + File currentDir = sd.getCurrentDir(); + List allLogFiles = matchEditLogs( + FileUtil.listFiles(currentDir)); + List ret = Lists.newArrayListWithCapacity( + allLogFiles.size()); + for (EditLogFile elf : allLogFiles) { + if (elf.isCorrupt() || elf.isInProgress()) continue; + if (elf.getFirstTxId() >= firstTxId) { + ret.add(new RemoteEditLog(elf.firstTxId, elf.lastTxId)); + } else if ((firstTxId > elf.getFirstTxId()) && + (firstTxId <= elf.getLastTxId())) { + throw new IOException("Asked for firstTxId " + firstTxId + + " which is in the middle of file " + elf.file); + } + } + + return ret; + } + + static List matchEditLogs(File[] filesInStorage) { + List ret = Lists.newArrayList(); + for (File f : filesInStorage) { + String name = f.getName(); + // Check for edits + Matcher editsMatch = EDITS_REGEX.matcher(name); + if (editsMatch.matches()) { + try { + long startTxId = Long.valueOf(editsMatch.group(1)); + long endTxId = Long.valueOf(editsMatch.group(2)); + ret.add(new EditLogFile(f, startTxId, endTxId)); + } catch (NumberFormatException nfe) { + LOG.error("Edits file " + f + " has improperly formatted " + + "transaction ID"); + // skip + } + } + + // Check for in-progress edits + Matcher inProgressEditsMatch = EDITS_INPROGRESS_REGEX.matcher(name); + if (inProgressEditsMatch.matches()) { + try { + long startTxId = Long.valueOf(inProgressEditsMatch.group(1)); + ret.add( + new EditLogFile(f, startTxId, EditLogFile.UNKNOWN_END)); + } catch (NumberFormatException nfe) { + LOG.error("In-progress edits file " + f + " has improperly " + + "formatted transaction ID"); + // skip + } + } + } + return ret; + } + + /** + * Record of an edit log that has been located and had its filename parsed. + */ + static class EditLogFile { + private File file; + private final long firstTxId; + private long lastTxId; + + private EditLogValidation cachedValidation = null; + private boolean isCorrupt = false; + + static final long UNKNOWN_END = -1; + + final static Comparator COMPARE_BY_START_TXID + = new Comparator() { + public int compare(EditLogFile a, EditLogFile b) { + return ComparisonChain.start() + .compare(a.getFirstTxId(), b.getFirstTxId()) + .compare(a.getLastTxId(), b.getLastTxId()) + .result(); + } + }; + + EditLogFile(File file, + long firstTxId, long lastTxId) { + assert lastTxId == UNKNOWN_END || lastTxId >= firstTxId; + assert firstTxId > 0; + assert file != null; + + this.firstTxId = firstTxId; + this.lastTxId = lastTxId; + this.file = file; + } + + public void finalizeLog() throws IOException { + long numTransactions = validateLog().numTransactions; + long lastTxId = firstTxId + numTransactions - 1; + File dst = new File(file.getParentFile(), + NNStorage.getFinalizedEditsFileName(firstTxId, lastTxId)); + LOG.info("Finalizing edits log " + file + " by renaming to " + + dst.getName()); + if (!file.renameTo(dst)) { + throw new IOException("Couldn't finalize log " + + file + " to " + dst); + } + this.lastTxId = lastTxId; + file = dst; + } + + long getFirstTxId() { + return firstTxId; + } + + long getLastTxId() { + return lastTxId; + } + + EditLogValidation validateLog() throws IOException { + if (cachedValidation == null) { + cachedValidation = EditLogFileInputStream.validateEditLog(file); + } + return cachedValidation; + } + + boolean isInProgress() { + return (lastTxId == UNKNOWN_END); + } + + File getFile() { + return file; + } + + void markCorrupt() { + isCorrupt = true; + } + + boolean isCorrupt() { + return isCorrupt; + } + + void moveAsideCorruptFile() throws IOException { + assert isCorrupt; + + File src = file; + File dst = new File(src.getParent(), src.getName() + ".corrupt"); + boolean success = src.renameTo(dst); + if (!success) { + throw new IOException( + "Couldn't rename corrupt log " + src + " to " + dst); + } + file = dst; + } + + @Override + public String toString() { + return String.format("EditLogFile(file=%s,first=%019d,last=%019d," + +"inProgress=%b,corrupt=%b)", file.toString(), + firstTxId, lastTxId, isInProgress(), isCorrupt); + } + } } diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FsckServlet.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FsckServlet.java index 96ccad6d422..3831b4580f6 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FsckServlet.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FsckServlet.java @@ -30,6 +30,7 @@ import javax.servlet.http.HttpServletResponse; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.protocol.FSConstants.DatanodeReportType; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.security.UserGroupInformation; /** @@ -59,13 +60,12 @@ public class FsckServlet extends DfsServlet { NameNode nn = NameNodeHttpServer.getNameNodeFromContext(context); final FSNamesystem namesystem = nn.getNamesystem(); + final BlockManager bm = namesystem.getBlockManager(); final int totalDatanodes = - namesystem.getNumberOfDatanodes(DatanodeReportType.LIVE); - final short minReplication = namesystem.getMinReplication(); - + namesystem.getNumberOfDatanodes(DatanodeReportType.LIVE); new NamenodeFsck(conf, nn, - NamenodeJspHelper.getNetworkTopology(nn), pmap, out, - totalDatanodes, minReplication, remoteAddress).fsck(); + bm.getDatanodeManager().getNetworkTopology(), pmap, out, + totalDatanodes, bm.minReplication, remoteAddress).fsck(); return null; } diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java index 56ea5c25129..d62aaa7e5ac 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java @@ -55,7 +55,7 @@ interface JournalManager { * @param purger the purging implementation to use * @throws IOException if purging fails */ - void purgeLogsOlderThan(long minTxIdToKeep, StoragePurger purger) + void purgeLogsOlderThan(long minTxIdToKeep) throws IOException; /** diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/ListPathsServlet.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/ListPathsServlet.java index aa0092b5eab..a3d2284cf79 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/ListPathsServlet.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/ListPathsServlet.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.server.common.JspHelper; +import org.apache.hadoop.util.ServletUtil; import org.apache.hadoop.util.VersionInfo; import org.znerd.xmlenc.*; @@ -86,8 +87,7 @@ public class ListPathsServlet extends DfsServlet { */ protected Map buildRoot(HttpServletRequest request, XMLOutputter doc) { - final String path = request.getPathInfo() != null - ? request.getPathInfo() : "/"; + final String path = ServletUtil.getDecodedPath(request, "/listPaths"); final String exclude = request.getParameter("exclude") != null ? request.getParameter("exclude") : ""; final String filter = request.getParameter("filter") != null @@ -135,6 +135,7 @@ public class ListPathsServlet extends DfsServlet { final Map root = buildRoot(request, doc); final String path = root.get("path"); + final String filePath = ServletUtil.getDecodedPath(request, "/listPaths"); try { final boolean recur = "yes".equals(root.get("recursive")); @@ -153,7 +154,7 @@ public class ListPathsServlet extends DfsServlet { doc.attribute(m.getKey(), m.getValue()); } - HdfsFileStatus base = nn.getFileInfo(path); + HdfsFileStatus base = nn.getFileInfo(filePath); if ((base != null) && base.isDir()) { writeInfo(base.getFullPath(new Path(path)), base, doc); } diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java index 4b5f9a90902..fe651001aa3 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java @@ -27,8 +27,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundEditLog; -import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundFSImage; +import org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.FSImageFile; +import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile; import org.apache.hadoop.hdfs.util.MD5FileUtils; import com.google.common.collect.Lists; @@ -80,14 +80,14 @@ public class NNStorageRetentionManager { // If fsimage_N is the image we want to keep, then we need to keep // all txns > N. We can remove anything < N+1, since fsimage_N // reflects the state up to and including N. - editLog.purgeLogsOlderThan(minImageTxId + 1, purger); + editLog.purgeLogsOlderThan(minImageTxId + 1); } private void purgeCheckpointsOlderThan( FSImageTransactionalStorageInspector inspector, long minTxId) { - for (FoundFSImage image : inspector.getFoundImages()) { - if (image.getTxId() < minTxId) { + for (FSImageFile image : inspector.getFoundImages()) { + if (image.getCheckpointTxId() < minTxId) { LOG.info("Purging old image " + image); purger.purgeImage(image); } @@ -101,10 +101,10 @@ public class NNStorageRetentionManager { */ private long getImageTxIdToRetain(FSImageTransactionalStorageInspector inspector) { - List images = inspector.getFoundImages(); + List images = inspector.getFoundImages(); TreeSet imageTxIds = Sets.newTreeSet(); - for (FoundFSImage image : images) { - imageTxIds.add(image.getTxId()); + for (FSImageFile image : images) { + imageTxIds.add(image.getCheckpointTxId()); } List imageTxIdsList = Lists.newArrayList(imageTxIds); @@ -124,18 +124,18 @@ public class NNStorageRetentionManager { * Interface responsible for disposing of old checkpoints and edit logs. */ static interface StoragePurger { - void purgeLog(FoundEditLog log); - void purgeImage(FoundFSImage image); + void purgeLog(EditLogFile log); + void purgeImage(FSImageFile image); } static class DeletionStoragePurger implements StoragePurger { @Override - public void purgeLog(FoundEditLog log) { + public void purgeLog(EditLogFile log) { deleteOrWarn(log.getFile()); } @Override - public void purgeImage(FoundFSImage image) { + public void purgeImage(FSImageFile image) { deleteOrWarn(image.getFile()); deleteOrWarn(MD5FileUtils.getDigestFileForFile(image.getFile())); } diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 4b227225d49..6fe89a67655 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -58,6 +58,11 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.FSConstants; +import org.apache.hadoop.hdfs.protocol.FSConstants.DatanodeReportType; +import org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction; +import org.apache.hadoop.hdfs.protocol.FSConstants.UpgradeAction; +import static org.apache.hadoop.hdfs.protocol.FSConstants.MAX_PATH_LENGTH; +import static org.apache.hadoop.hdfs.protocol.FSConstants.MAX_PATH_DEPTH; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; @@ -146,7 +151,7 @@ import org.apache.hadoop.util.StringUtils; * NameNode state, for example partial blocksMap etc. **********************************************************/ @InterfaceAudience.Private -public class NameNode implements NamenodeProtocols, FSConstants { +public class NameNode implements NamenodeProtocols { static{ HdfsConfiguration.init(); } @@ -654,7 +659,7 @@ public class NameNode implements NamenodeProtocols, FSConstants { "Unexpected not positive size: "+size); } - return namesystem.getBlocks(datanode, size); + return namesystem.getBlockManager().getBlocks(datanode, size); } @Override // NamenodeProtocol @@ -750,8 +755,8 @@ public class NameNode implements NamenodeProtocols, FSConstants { +src+" for "+clientName+" at "+clientMachine); } if (!checkPathLength(src)) { - throw new IOException("create: Pathname too long. Limit " - + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels."); + throw new IOException("create: Pathname too long. Limit " + + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels."); } namesystem.startFile(src, new PermissionStatus(UserGroupInformation.getCurrentUser().getShortUserName(), @@ -898,7 +903,7 @@ public class NameNode implements NamenodeProtocols, FSConstants { DatanodeInfo[] nodes = blocks[i].getLocations(); for (int j = 0; j < nodes.length; j++) { DatanodeInfo dn = nodes[j]; - namesystem.markBlockAsCorrupt(blk, dn); + namesystem.getBlockManager().findAndMarkBlockAsCorrupt(blk, dn); } } } @@ -944,8 +949,8 @@ public class NameNode implements NamenodeProtocols, FSConstants { stateChangeLog.debug("*DIR* NameNode.rename: " + src + " to " + dst); } if (!checkPathLength(dst)) { - throw new IOException("rename: Pathname too long. Limit " - + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels."); + throw new IOException("rename: Pathname too long. Limit " + + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels."); } boolean ret = namesystem.renameTo(src, dst); if (ret) { @@ -968,8 +973,8 @@ public class NameNode implements NamenodeProtocols, FSConstants { stateChangeLog.debug("*DIR* NameNode.rename: " + src + " to " + dst); } if (!checkPathLength(dst)) { - throw new IOException("rename: Pathname too long. Limit " - + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels."); + throw new IOException("rename: Pathname too long. Limit " + + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels."); } namesystem.renameTo(src, dst, options); metrics.incrFilesRenamed(); @@ -1100,7 +1105,8 @@ public class NameNode implements NamenodeProtocols, FSConstants { @Override // ClientProtocol public void refreshNodes() throws IOException { // TODO:HA decide on OperationCategory for this - namesystem.refreshNodes(new HdfsConfiguration()); + namesystem.getBlockManager().getDatanodeManager().refreshNodes( + new HdfsConfiguration()); } @Override // NamenodeProtocol @@ -1119,7 +1125,7 @@ public class NameNode implements NamenodeProtocols, FSConstants { public RemoteEditLogManifest getEditLogManifest(long sinceTxId) throws IOException { // TODO:HA decide on OperationCategory for this - return namesystem.getEditLogManifest(sinceTxId); + return namesystem.getEditLog().getEditLogManifest(sinceTxId); } @Override // ClientProtocol @@ -1167,7 +1173,7 @@ public class NameNode implements NamenodeProtocols, FSConstants { @Override // ClientProtocol public void setBalancerBandwidth(long bandwidth) throws IOException { // TODO:HA decide on OperationCategory for this - namesystem.setBalancerBandwidth(bandwidth); + namesystem.getBlockManager().getDatanodeManager().setBalancerBandwidth(bandwidth); } @Override // ClientProtocol @@ -1271,7 +1277,7 @@ public class NameNode implements NamenodeProtocols, FSConstants { + " blocks"); } - namesystem.processReport(nodeReg, poolId, blist); + namesystem.getBlockManager().processReport(nodeReg, poolId, blist); if (getFSImage().isUpgradeFinalized()) return new DatanodeCommand.Finalize(poolId); return null; @@ -1286,7 +1292,8 @@ public class NameNode implements NamenodeProtocols, FSConstants { +"from "+nodeReg.getName()+" "+blocks.length+" blocks."); } for (int i = 0; i < blocks.length; i++) { - namesystem.blockReceived(nodeReg, poolId, blocks[i], delHints[i]); + namesystem.getBlockManager().blockReceived( + nodeReg, poolId, blocks[i], delHints[i]); } } @@ -1305,7 +1312,7 @@ public class NameNode implements NamenodeProtocols, FSConstants { LOG.warn("Disk error on " + dnName + ": " + msg); } else if (errorCode == DatanodeProtocol.FATAL_DISK_ERROR) { LOG.warn("Fatal disk error on " + dnName + ": " + msg); - namesystem.removeDatanode(nodeReg); + namesystem.getBlockManager().getDatanodeManager().removeDatanode(nodeReg); } else { LOG.info("Error report from " + dnName + ": " + msg); } @@ -1347,7 +1354,7 @@ public class NameNode implements NamenodeProtocols, FSConstants { * @throws IOException */ public void verifyVersion(int version) throws IOException { - if (version != LAYOUT_VERSION) + if (version != FSConstants.LAYOUT_VERSION) throw new IncorrectVersionException(version, "data node"); } diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java index c2db25cb168..f9ba6d4a419 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java @@ -646,46 +646,77 @@ public class NamenodeFsck { /** {@inheritDoc} */ public String toString() { StringBuilder res = new StringBuilder(); - res.append("Status: " + (isHealthy() ? "HEALTHY" : "CORRUPT")); - res.append("\n Total size:\t" + totalSize + " B"); - if (totalOpenFilesSize != 0) - res.append(" (Total open files size: " + totalOpenFilesSize + " B)"); - res.append("\n Total dirs:\t" + totalDirs); - res.append("\n Total files:\t" + totalFiles); - if (totalOpenFiles != 0) - res.append(" (Files currently being written: " + - totalOpenFiles + ")"); - res.append("\n Total blocks (validated):\t" + totalBlocks); - if (totalBlocks > 0) res.append(" (avg. block size " - + (totalSize / totalBlocks) + " B)"); - if (totalOpenFilesBlocks != 0) - res.append(" (Total open file blocks (not validated): " + - totalOpenFilesBlocks + ")"); - if (corruptFiles > 0) { - res.append("\n ********************************"); - res.append("\n CORRUPT FILES:\t" + corruptFiles); + res.append("Status: ").append((isHealthy() ? "HEALTHY" : "CORRUPT")) + .append("\n Total size:\t").append(totalSize).append(" B"); + if (totalOpenFilesSize != 0) { + res.append(" (Total open files size: ").append(totalOpenFilesSize) + .append(" B)"); + } + res.append("\n Total dirs:\t").append(totalDirs).append( + "\n Total files:\t").append(totalFiles); + if (totalOpenFiles != 0) { + res.append(" (Files currently being written: ").append(totalOpenFiles) + .append(")"); + } + res.append("\n Total blocks (validated):\t").append(totalBlocks); + if (totalBlocks > 0) { + res.append(" (avg. block size ").append((totalSize / totalBlocks)) + .append(" B)"); + } + if (totalOpenFilesBlocks != 0) { + res.append(" (Total open file blocks (not validated): ").append( + totalOpenFilesBlocks).append(")"); + } + if (corruptFiles > 0) { + res.append("\n ********************************").append( + "\n CORRUPT FILES:\t").append(corruptFiles); if (missingSize > 0) { - res.append("\n MISSING BLOCKS:\t" + missingIds.size()); - res.append("\n MISSING SIZE:\t\t" + missingSize + " B"); + res.append("\n MISSING BLOCKS:\t").append(missingIds.size()).append( + "\n MISSING SIZE:\t\t").append(missingSize).append(" B"); } if (corruptBlocks > 0) { - res.append("\n CORRUPT BLOCKS: \t" + corruptBlocks); + res.append("\n CORRUPT BLOCKS: \t").append(corruptBlocks); } res.append("\n ********************************"); } - res.append("\n Minimally replicated blocks:\t" + numMinReplicatedBlocks); - if (totalBlocks > 0) res.append(" (" + ((float) (numMinReplicatedBlocks * 100) / (float) totalBlocks) + " %)"); - res.append("\n Over-replicated blocks:\t" + numOverReplicatedBlocks); - if (totalBlocks > 0) res.append(" (" + ((float) (numOverReplicatedBlocks * 100) / (float) totalBlocks) + " %)"); - res.append("\n Under-replicated blocks:\t" + numUnderReplicatedBlocks); - if (totalBlocks > 0) res.append(" (" + ((float) (numUnderReplicatedBlocks * 100) / (float) totalBlocks) + " %)"); - res.append("\n Mis-replicated blocks:\t\t" + numMisReplicatedBlocks); - if (totalBlocks > 0) res.append(" (" + ((float) (numMisReplicatedBlocks * 100) / (float) totalBlocks) + " %)"); - res.append("\n Default replication factor:\t" + replication); - res.append("\n Average block replication:\t" + getReplicationFactor()); - res.append("\n Corrupt blocks:\t\t" + corruptBlocks); - res.append("\n Missing replicas:\t\t" + missingReplicas); - if (totalReplicas > 0) res.append(" (" + ((float) (missingReplicas * 100) / (float) totalReplicas) + " %)"); + res.append("\n Minimally replicated blocks:\t").append( + numMinReplicatedBlocks); + if (totalBlocks > 0) { + res.append(" (").append( + ((float) (numMinReplicatedBlocks * 100) / (float) totalBlocks)) + .append(" %)"); + } + res.append("\n Over-replicated blocks:\t") + .append(numOverReplicatedBlocks); + if (totalBlocks > 0) { + res.append(" (").append( + ((float) (numOverReplicatedBlocks * 100) / (float) totalBlocks)) + .append(" %)"); + } + res.append("\n Under-replicated blocks:\t").append( + numUnderReplicatedBlocks); + if (totalBlocks > 0) { + res.append(" (").append( + ((float) (numUnderReplicatedBlocks * 100) / (float) totalBlocks)) + .append(" %)"); + } + res.append("\n Mis-replicated blocks:\t\t") + .append(numMisReplicatedBlocks); + if (totalBlocks > 0) { + res.append(" (").append( + ((float) (numMisReplicatedBlocks * 100) / (float) totalBlocks)) + .append(" %)"); + } + res.append("\n Default replication factor:\t").append(replication) + .append("\n Average block replication:\t").append( + getReplicationFactor()).append("\n Corrupt blocks:\t\t").append( + corruptBlocks).append("\n Missing replicas:\t\t").append( + missingReplicas); + if (totalReplicas > 0) { + res.append(" (").append( + ((float) (missingReplicas * 100) / (float) totalReplicas)).append( + " %)"); + } return res.toString(); } } diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java index d28e3307d90..f50e1f8b9f3 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java @@ -42,13 +42,14 @@ import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.FSConstants.UpgradeAction; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport; import org.apache.hadoop.io.Text; -import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.net.NodeBase; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; @@ -229,14 +230,10 @@ class NamenodeJspHelper { void generateHealthReport(JspWriter out, NameNode nn, HttpServletRequest request) throws IOException { FSNamesystem fsn = nn.getNamesystem(); - ArrayList live = new ArrayList(); - ArrayList dead = new ArrayList(); - fsn.DFSNodesStatus(live, dead); - // If a data node has been first included in the include list, - // then decommissioned, then removed from both include and exclude list. - // We make the web console to "forget" this node by not displaying it. - fsn.removeDecomNodeFromList(live); - fsn.removeDecomNodeFromList(dead); + final DatanodeManager dm = fsn.getBlockManager().getDatanodeManager(); + final List live = new ArrayList(); + final List dead = new ArrayList(); + dm.fetchDatanodes(live, dead, true); int liveDecommissioned = 0; for (DatanodeDescriptor d : live) { @@ -248,8 +245,7 @@ class NamenodeJspHelper { deadDecommissioned += d.isDecommissioned() ? 1 : 0; } - ArrayList decommissioning = fsn - .getDecommissioningNodes(); + final List decommissioning = dm.getDecommissioningNodes(); sorterField = request.getParameter("sorter/field"); sorterOrder = request.getParameter("sorter/order"); @@ -349,7 +345,7 @@ class NamenodeJspHelper { + colTxt() + ":" + colTxt() + decommissioning.size() + rowTxt() + colTxt("Excludes missing blocks.") + "Number of Under-Replicated Blocks" + colTxt() + ":" + colTxt() - + fsn.getUnderReplicatedNotMissingBlocks() + + fsn.getBlockManager().getUnderReplicatedNotMissingBlocks() + "
    \n"); if (live.isEmpty() && dead.isEmpty()) { @@ -370,15 +366,10 @@ class NamenodeJspHelper { return token == null ? null : token.encodeToUrlString(); } - /** @return the network topology. */ - static NetworkTopology getNetworkTopology(final NameNode namenode) { - return namenode.getNamesystem().getBlockManager().getDatanodeManager( - ).getNetworkTopology(); - } - /** @return a randomly chosen datanode. */ static DatanodeDescriptor getRandomDatanode(final NameNode namenode) { - return (DatanodeDescriptor)getNetworkTopology(namenode).chooseRandom( + return (DatanodeDescriptor)namenode.getNamesystem().getBlockManager( + ).getDatanodeManager().getNetworkTopology().chooseRandom( NodeBase.ROOT); } @@ -564,12 +555,14 @@ class NamenodeJspHelper { void generateNodesList(ServletContext context, JspWriter out, HttpServletRequest request) throws IOException { - ArrayList live = new ArrayList(); - ArrayList dead = new ArrayList(); final NameNode nn = NameNodeHttpServer.getNameNodeFromContext(context); - nn.getNamesystem().DFSNodesStatus(live, dead); - nn.getNamesystem().removeDecomNodeFromList(live); - nn.getNamesystem().removeDecomNodeFromList(dead); + final FSNamesystem ns = nn.getNamesystem(); + final DatanodeManager dm = ns.getBlockManager().getDatanodeManager(); + + final List live = new ArrayList(); + final List dead = new ArrayList(); + dm.fetchDatanodes(live, dead, true); + InetSocketAddress nnSocketAddress = (InetSocketAddress) context .getAttribute(NameNodeHttpServer.NAMENODE_ADDRESS_ATTRIBUTE_KEY); String nnaddr = nnSocketAddress.getAddress().getHostAddress() + ":" @@ -678,8 +671,7 @@ class NamenodeJspHelper { } } else if (whatNodes.equals("DECOMMISSIONING")) { // Decommissioning Nodes - ArrayList decommissioning = nn.getNamesystem() - .getDecommissioningNodes(); + final List decommissioning = dm.getDecommissioningNodes(); out.print("
    " + " Decommissioning Datanodes : " + decommissioning.size() + "

    \n"); @@ -715,16 +707,17 @@ class NamenodeJspHelper { static class XMLBlockInfo { final Block block; final INodeFile inode; - final FSNamesystem fsn; + final BlockManager blockManager; - public XMLBlockInfo(FSNamesystem fsn, Long blockId) { - this.fsn = fsn; + XMLBlockInfo(FSNamesystem fsn, Long blockId) { + this.blockManager = fsn.getBlockManager(); + if (blockId == null) { this.block = null; this.inode = null; } else { this.block = new Block(blockId); - this.inode = fsn.getBlockManager().getINode(block); + this.inode = blockManager.getINode(block); } } @@ -798,31 +791,25 @@ class NamenodeJspHelper { } doc.startTag("replicas"); - - if (fsn.getBlockManager().blocksMap.contains(block)) { - Iterator it = - fsn.getBlockManager().blocksMap.nodeIterator(block); + for(final Iterator it = blockManager.datanodeIterator(block); + it.hasNext(); ) { + doc.startTag("replica"); - while (it.hasNext()) { - doc.startTag("replica"); + DatanodeDescriptor dd = it.next(); - DatanodeDescriptor dd = it.next(); + doc.startTag("host_name"); + doc.pcdata(dd.getHostName()); + doc.endTag(); - doc.startTag("host_name"); - doc.pcdata(dd.getHostName()); - doc.endTag(); - - boolean isCorrupt = fsn.getCorruptReplicaBlockIds(0, - block.getBlockId()) != null; - - doc.startTag("is_corrupt"); - doc.pcdata(""+isCorrupt); - doc.endTag(); - - doc.endTag(); // - } - - } + boolean isCorrupt = blockManager.getCorruptReplicaBlockIds(0, + block.getBlockId()) != null; + + doc.startTag("is_corrupt"); + doc.pcdata(""+isCorrupt); + doc.endTag(); + + doc.endTag(); // + } doc.endTag(); // } @@ -834,14 +821,14 @@ class NamenodeJspHelper { // utility class used in corrupt_replicas_xml.jsp static class XMLCorruptBlockInfo { - final FSNamesystem fsn; final Configuration conf; final Long startingBlockId; final int numCorruptBlocks; + final BlockManager blockManager; - public XMLCorruptBlockInfo(FSNamesystem fsn, Configuration conf, + XMLCorruptBlockInfo(FSNamesystem fsn, Configuration conf, int numCorruptBlocks, Long startingBlockId) { - this.fsn = fsn; + this.blockManager = fsn.getBlockManager(); this.conf = conf; this.numCorruptBlocks = numCorruptBlocks; this.startingBlockId = startingBlockId; @@ -864,17 +851,16 @@ class NamenodeJspHelper { doc.endTag(); doc.startTag("num_missing_blocks"); - doc.pcdata(""+fsn.getMissingBlocksCount()); + doc.pcdata(""+blockManager.getMissingBlocksCount()); doc.endTag(); doc.startTag("num_corrupt_replica_blocks"); - doc.pcdata(""+fsn.getCorruptReplicaBlocks()); + doc.pcdata(""+blockManager.getCorruptReplicaBlocksCount()); doc.endTag(); doc.startTag("corrupt_replica_block_ids"); - long[] corruptBlockIds - = fsn.getCorruptReplicaBlockIds(numCorruptBlocks, - startingBlockId); + final long[] corruptBlockIds = blockManager.getCorruptReplicaBlockIds( + numCorruptBlocks, startingBlockId); if (corruptBlockIds != null) { for (Long blockId: corruptBlockIds) { doc.startTag("block_id"); diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/StreamFile.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/StreamFile.java index c56f3913f3e..c93a899a315 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/StreamFile.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/StreamFile.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DatanodeJspHelper; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.ServletUtil; import org.mortbay.jetty.InclusiveByteRange; @InterfaceAudience.Private @@ -57,13 +58,14 @@ public class StreamFile extends DfsServlet { final DataNode datanode = (DataNode) context.getAttribute("datanode"); return DatanodeJspHelper.getDFSClient(request, datanode, conf, ugi); } - + @SuppressWarnings("unchecked") public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - final String path = request.getPathInfo() != null ? - request.getPathInfo() : "/"; + final String path = ServletUtil.getDecodedPath(request, "/streamFile"); + final String rawPath = ServletUtil.getRawPath(request, "/streamFile"); final String filename = JspHelper.validatePath(path); + final String rawFilename = JspHelper.validatePath(rawPath); if (filename == null) { response.setContentType("text/plain"); PrintWriter out = response.getWriter(); @@ -98,7 +100,7 @@ public class StreamFile extends DfsServlet { } else { // No ranges, so send entire file response.setHeader("Content-Disposition", "attachment; filename=\"" + - filename + "\""); + rawFilename + "\""); response.setContentType("application/octet-stream"); response.setHeader(CONTENT_LENGTH, "" + fileLen); StreamFile.copyFromOffset(in, out, 0L, fileLen); diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java index 302e9743be3..944e998ecfd 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java @@ -41,7 +41,7 @@ import com.google.common.collect.Lists; /** * This class provides fetching a specified file from the NameNode. */ -class TransferFsImage implements FSConstants { +class TransferFsImage { public final static String CONTENT_LENGTH = "Content-Length"; public final static String MD5_HEADER = "X-MD5-Digest"; @@ -69,6 +69,8 @@ class TransferFsImage implements FSConstants { static void downloadEditsToStorage(String fsName, RemoteEditLog log, NNStorage dstStorage) throws IOException { + assert log.getStartTxId() > 0 && log.getEndTxId() > 0 : + "bad log: " + log; String fileid = GetImageServlet.getParamStringForLog( log, dstStorage); String fileName = NNStorage.getFinalizedEditsFileName( @@ -122,7 +124,7 @@ class TransferFsImage implements FSConstants { static void getFileServer(OutputStream outstream, File localfile, DataTransferThrottler throttler) throws IOException { - byte buf[] = new byte[BUFFER_SIZE]; + byte buf[] = new byte[FSConstants.IO_FILE_BUFFER_SIZE]; FileInputStream infile = null; try { infile = new FileInputStream(localfile); @@ -137,7 +139,7 @@ class TransferFsImage implements FSConstants { && localfile.getAbsolutePath().contains("fsimage")) { // Test sending image shorter than localfile long len = localfile.length(); - buf = new byte[(int)Math.min(len/2, BUFFER_SIZE)]; + buf = new byte[(int)Math.min(len/2, FSConstants.IO_FILE_BUFFER_SIZE)]; // This will read at most half of the image // and the rest of the image will be sent over the wire infile.read(buf); @@ -177,7 +179,7 @@ class TransferFsImage implements FSConstants { static MD5Hash getFileClient(String nnHostPort, String queryString, List localPaths, NNStorage dstStorage, boolean getChecksum) throws IOException { - byte[] buf = new byte[BUFFER_SIZE]; + byte[] buf = new byte[FSConstants.IO_FILE_BUFFER_SIZE]; String proto = UserGroupInformation.isSecurityEnabled() ? "https://" : "http://"; StringBuilder str = new StringBuilder(proto+nnHostPort+"/getimage?"); str.append(queryString); diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLog.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLog.java index 5801961264f..5b8ac59f37e 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLog.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLog.java @@ -20,11 +20,15 @@ package org.apache.hadoop.hdfs.server.protocol; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.util.Comparator; import org.apache.hadoop.hdfs.protocol.FSConstants; import org.apache.hadoop.io.Writable; -public class RemoteEditLog implements Writable { +import com.google.common.base.Function; +import com.google.common.collect.ComparisonChain; + +public class RemoteEditLog implements Writable, Comparable { private long startTxId = FSConstants.INVALID_TXID; private long endTxId = FSConstants.INVALID_TXID; @@ -60,5 +64,34 @@ public class RemoteEditLog implements Writable { startTxId = in.readLong(); endTxId = in.readLong(); } + + @Override + public int compareTo(RemoteEditLog log) { + return ComparisonChain.start() + .compare(startTxId, log.startTxId) + .compare(endTxId, log.endTxId) + .result(); + } + @Override + public boolean equals(Object o) { + if (!(o instanceof RemoteEditLog)) return false; + return this.compareTo((RemoteEditLog)o) == 0; + } + + @Override + public int hashCode() { + return (int) (startTxId * endTxId); + } + + /** + * Guava Function which applies {@link #getStartTxId()} + */ + public static final Function GET_START_TXID = + new Function() { + @Override + public Long apply(RemoteEditLog log) { + return log.getStartTxId(); + } + }; } diff --git a/hadoop-common/src/main/java/org/apache/hadoop/util/CyclicIteration.java b/hdfs/src/java/org/apache/hadoop/hdfs/util/CyclicIteration.java similarity index 98% rename from hadoop-common/src/main/java/org/apache/hadoop/util/CyclicIteration.java rename to hdfs/src/java/org/apache/hadoop/hdfs/util/CyclicIteration.java index a3b27b4e16d..78c385eab2a 100644 --- a/hadoop-common/src/main/java/org/apache/hadoop/util/CyclicIteration.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/util/CyclicIteration.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.util; +package org.apache.hadoop.hdfs.util; import java.util.Iterator; import java.util.Map; @@ -110,4 +110,4 @@ public class CyclicIteration implements Iterable> { throw new UnsupportedOperationException("Not supported"); } } -} \ No newline at end of file +} diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/util/Holder.java b/hdfs/src/java/org/apache/hadoop/hdfs/util/Holder.java new file mode 100644 index 00000000000..1832055940f --- /dev/null +++ b/hdfs/src/java/org/apache/hadoop/hdfs/util/Holder.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.util; + +/** + * A Holder is simply a wrapper around some other object. This is useful + * in particular for storing immutable values like boxed Integers in a + * collection without having to do the "lookup" of the value twice. + */ +public class Holder { + public T held; + + public Holder(T held) { + this.held = held; + } + + @Override + public String toString() { + return String.valueOf(held); + } +} diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/util/RwLock.java b/hdfs/src/java/org/apache/hadoop/hdfs/util/RwLock.java new file mode 100644 index 00000000000..cd88963e3da --- /dev/null +++ b/hdfs/src/java/org/apache/hadoop/hdfs/util/RwLock.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.util; + +/** Read-write lock interface. */ +public interface RwLock { + /** Acquire read lock. */ + public void readLock(); + + /** Release read lock. */ + public void readUnlock(); + + /** Check if the current thread holds read lock. */ + public boolean hasReadLock(); + + /** Acquire write lock. */ + public void writeLock(); + + /** Release write lock. */ + public void writeUnlock(); + + /** Check if the current thread holds write lock. */ + public boolean hasWriteLock(); + + /** Check if the current thread holds read or write lock. */ + public boolean hasReadOrWriteLock(); +} diff --git a/hdfs/src/test/aop/org/apache/hadoop/hdfs/server/namenode/FileDataServletAspects.aj b/hdfs/src/test/aop/org/apache/hadoop/hdfs/server/namenode/FileDataServletAspects.aj index cbe34a8695b..6e74bca549c 100644 --- a/hdfs/src/test/aop/org/apache/hadoop/hdfs/server/namenode/FileDataServletAspects.aj +++ b/hdfs/src/test/aop/org/apache/hadoop/hdfs/server/namenode/FileDataServletAspects.aj @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import java.net.URI; -import java.net.URISyntaxException; +import java.net.URL; +import java.io.IOException; import javax.servlet.http.HttpServletRequest; @@ -30,18 +30,17 @@ import org.apache.hadoop.security.UserGroupInformation; public aspect FileDataServletAspects { static final Log LOG = FileDataServlet.LOG; - pointcut callCreateUri() : call (URI FileDataServlet.createUri( - String, HdfsFileStatus, UserGroupInformation, ClientProtocol, + pointcut callCreateUrl() : call (URL FileDataServlet.createRedirectURL( + String, String, HdfsFileStatus, UserGroupInformation, ClientProtocol, HttpServletRequest, String)); /** Replace host name with "localhost" for unit test environment. */ - URI around () throws URISyntaxException : callCreateUri() { - final URI original = proceed(); - LOG.info("FI: original uri = " + original); - final URI replaced = new URI(original.getScheme(), original.getUserInfo(), - "localhost", original.getPort(), original.getPath(), - original.getQuery(), original.getFragment()) ; - LOG.info("FI: replaced uri = " + replaced); + URL around () throws IOException : callCreateUrl() { + final URL original = proceed(); + LOG.info("FI: original url = " + original); + final URL replaced = new URL("http", "localhost", original.getPort(), + original.getPath() + '?' + original.getQuery()); + LOG.info("FI: replaced url = " + replaced); return replaced; } } diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/fs/TestResolveHdfsSymlink.java b/hdfs/src/test/hdfs/org/apache/hadoop/fs/TestResolveHdfsSymlink.java index 0c17dc62a0a..3b67f1b4d31 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/fs/TestResolveHdfsSymlink.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/fs/TestResolveHdfsSymlink.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; +import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; @@ -47,8 +48,8 @@ public class TestResolveHdfsSymlink { public static void setUp() throws IOException { Configuration conf = new HdfsConfiguration(); cluster = new MiniDFSCluster.Builder(conf).build(); - cluster.getNamesystem().getDelegationTokenSecretManager().startThreads(); cluster.waitActive(); + NameNodeAdapter.getDtSecretManager(cluster.getNamesystem()).startThreads(); } @AfterClass diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java b/hdfs/src/test/hdfs/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java index fd31b857d28..66b8fa32a3e 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java @@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; @@ -52,7 +53,7 @@ public class TestViewFileSystemHdfs extends ViewFileSystemBaseTest { SupportsBlocks = true; cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(2).build(); cluster.waitClusterUp(); - cluster.getNamesystem().getDelegationTokenSecretManager().startThreads(); + NameNodeAdapter.getDtSecretManager(cluster.getNamesystem()).startThreads(); fHdfs = cluster.getFileSystem(); defaultWorkingDirectory = fHdfs.makeQualified( new Path("/user/" + UserGroupInformation.getCurrentUser().getShortUserName())); diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/fs/viewfs/TestViewFsHdfs.java b/hdfs/src/test/hdfs/org/apache/hadoop/fs/viewfs/TestViewFsHdfs.java index a1f3b8f989c..4a60556a43f 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/fs/viewfs/TestViewFsHdfs.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/fs/viewfs/TestViewFsHdfs.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; @@ -52,7 +53,7 @@ public class TestViewFsHdfs extends ViewFsBaseTest { SupportsBlocks = true; cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(2).build(); cluster.waitClusterUp(); - cluster.getNamesystem().getDelegationTokenSecretManager().startThreads(); + NameNodeAdapter.getDtSecretManager(cluster.getNamesystem()).startThreads(); fc = FileContext.getFileContext(cluster.getURI(0), CONF); defaultWorkingDirectory = fc.makeQualified( new Path("/user/" + UserGroupInformation.getCurrentUser().getShortUserName())); diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java index 1d6b2d9fbf6..a3042cafffb 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -54,6 +54,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.protocol.FSConstants; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.datatransfer.Sender; @@ -61,6 +62,7 @@ import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BlockOpResponseP import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.TestTransferRbw; @@ -375,10 +377,9 @@ public class DFSTestUtil { /* * Return the total capacity of all live DNs. */ - public static long getLiveDatanodeCapacity(FSNamesystem ns) { - ArrayList live = new ArrayList(); - ArrayList dead = new ArrayList(); - ns.DFSNodesStatus(live, dead); + public static long getLiveDatanodeCapacity(DatanodeManager dm) { + final List live = new ArrayList(); + dm.fetchDatanodes(live, null, false); long capacity = 0; for (final DatanodeDescriptor dn : live) { capacity += dn.getCapacity(); @@ -389,21 +390,20 @@ public class DFSTestUtil { /* * Return the capacity of the given live DN. */ - public static long getDatanodeCapacity(FSNamesystem ns, int index) { - ArrayList live = new ArrayList(); - ArrayList dead = new ArrayList(); - ns.DFSNodesStatus(live, dead); + public static long getDatanodeCapacity(DatanodeManager dm, int index) { + final List live = new ArrayList(); + dm.fetchDatanodes(live, null, false); return live.get(index).getCapacity(); } /* * Wait for the given # live/dead DNs, total capacity, and # vol failures. */ - public static void waitForDatanodeStatus(FSNamesystem ns, int expectedLive, + public static void waitForDatanodeStatus(DatanodeManager dm, int expectedLive, int expectedDead, long expectedVolFails, long expectedTotalCapacity, long timeout) throws InterruptedException, TimeoutException { - ArrayList live = new ArrayList(); - ArrayList dead = new ArrayList(); + final List live = new ArrayList(); + final List dead = new ArrayList(); final int ATTEMPTS = 10; int count = 0; long currTotalCapacity = 0; @@ -413,7 +413,7 @@ public class DFSTestUtil { Thread.sleep(timeout); live.clear(); dead.clear(); - ns.DFSNodesStatus(live, dead); + dm.fetchDatanodes(live, dead, false); currTotalCapacity = 0; volFails = 0; for (final DatanodeDescriptor dd : live) { @@ -670,7 +670,7 @@ public class DFSTestUtil { final long writeTimeout = dfsClient.getDatanodeWriteTimeout(datanodes.length); final DataOutputStream out = new DataOutputStream(new BufferedOutputStream( NetUtils.getOutputStream(s, writeTimeout), - DataNode.SMALL_BUFFER_SIZE)); + FSConstants.SMALL_BUFFER_SIZE)); final DataInputStream in = new DataInputStream(NetUtils.getInputStream(s)); // send the request diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java index 4078248b614..89627b71b94 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -48,13 +48,13 @@ import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.FSConstants.DatanodeReportType; -import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption; +import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; import org.apache.hadoop.hdfs.server.datanode.DataStorage; import org.apache.hadoop.hdfs.server.datanode.FSDatasetInterface; import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset; -import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; @@ -71,7 +71,6 @@ import org.apache.hadoop.security.RefreshUserMappingsProtocol; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol; -import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.tools.GetUserMappingsProtocol; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.ToolRunner; @@ -1659,9 +1658,7 @@ public class MiniDFSCluster { * Set the softLimit and hardLimit of client lease periods */ public void setLeasePeriod(long soft, long hard) { - final FSNamesystem namesystem = getNamesystem(); - namesystem.leaseManager.setLeasePeriod(soft, hard); - namesystem.lmthread.interrupt(); + NameNodeAdapter.setLeasePeriod(getNamesystem(), soft, hard); } /** diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java index e013a913750..e3d6e171782 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java @@ -26,6 +26,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; /** * This class tests DatanodeDescriptor.getBlocksScheduled() at the @@ -50,7 +51,9 @@ public class TestBlocksScheduledCounter extends TestCase { ((DFSOutputStream)(out.getWrappedStream())).hflush(); ArrayList dnList = new ArrayList(); - cluster.getNamesystem().DFSNodesStatus(dnList, dnList); + final DatanodeManager dm = cluster.getNamesystem().getBlockManager( + ).getDatanodeManager(); + dm.fetchDatanodes(dnList, dnList, false); DatanodeDescriptor dn = dnList.get(0); assertEquals(1, dn.getBlocksScheduled()); diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRemove.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRemove.java index 86e0ea55165..8a8d404658e 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRemove.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRemove.java @@ -17,20 +17,16 @@ */ package org.apache.hadoop.hdfs; -import java.io.*; +import java.io.DataOutputStream; +import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.protocol.FSConstants; import org.apache.hadoop.hdfs.server.datanode.DataNode; public class TestDFSRemove extends junit.framework.TestCase { - static int countLease(MiniDFSCluster cluster) { - return cluster.getNamesystem().leaseManager.countLease(); - } - final Path dir = new Path("/test/remove/"); void list(FileSystem fs, String name) throws IOException { @@ -76,7 +72,7 @@ public class TestDFSRemove extends junit.framework.TestCase { fs.delete(a, false); } // wait 3 heartbeat intervals, so that all blocks are deleted. - Thread.sleep(3 * FSConstants.HEARTBEAT_INTERVAL * 1000); + Thread.sleep(3 * DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT * 1000); // all blocks should be gone now. long dfsUsedFinal = getTotalDfsUsed(cluster); assertEquals("All blocks should be gone. start=" + dfsUsedStart diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRename.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRename.java index 0d7cb6581ac..ce1c62b48b3 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRename.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRename.java @@ -17,16 +17,18 @@ */ package org.apache.hadoop.hdfs; -import java.io.*; +import java.io.DataOutputStream; +import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; public class TestDFSRename extends junit.framework.TestCase { static int countLease(MiniDFSCluster cluster) { - return cluster.getNamesystem().leaseManager.countLease(); + return NameNodeAdapter.getLeaseManager(cluster.getNamesystem()).countLease(); } final Path dir = new Path("/test/rename/"); diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestDecommission.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestDecommission.java index 2a88f78688a..7ef901de743 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestDecommission.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestDecommission.java @@ -17,6 +17,10 @@ */ package org.apache.hadoop.hdfs; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + import java.io.IOException; import java.util.ArrayList; import java.util.Collection; @@ -30,14 +34,12 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; -import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates; import org.apache.hadoop.hdfs.protocol.FSConstants.DatanodeReportType; +import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; -import static org.junit.Assert.*; - import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -224,7 +226,7 @@ public class TestDecommission { } nodes.add(nodename); writeConfigFile(excludeFile, nodes); - cluster.getNamesystem(nnIndex).refreshNodes(conf); + refreshNodes(cluster.getNamesystem(nnIndex), conf); DatanodeInfo ret = NameNodeAdapter.getDatanode( cluster.getNamesystem(nnIndex), info[index]); waitNodeState(ret, waitForState); @@ -235,7 +237,7 @@ public class TestDecommission { private void recomissionNode(DatanodeInfo decommissionedNode) throws IOException { LOG.info("Recommissioning node: " + decommissionedNode.getName()); writeConfigFile(excludeFile, null); - cluster.getNamesystem().refreshNodes(conf); + refreshNodes(cluster.getNamesystem(), conf); waitNodeState(decommissionedNode, AdminStates.NORMAL); } @@ -284,6 +286,11 @@ public class TestDecommission { validateCluster(client, numDatanodes); } } + + static void refreshNodes(final FSNamesystem ns, final Configuration conf + ) throws IOException { + ns.getBlockManager().getDatanodeManager().refreshNodes(conf); + } private void verifyStats(NameNode namenode, FSNamesystem fsn, DatanodeInfo node, boolean decommissioning) throws InterruptedException { @@ -465,7 +472,7 @@ public class TestDecommission { // Stop decommissioning and verify stats writeConfigFile(excludeFile, null); - fsn.refreshNodes(conf); + refreshNodes(fsn, conf); DatanodeInfo ret = NameNodeAdapter.getDatanode(fsn, downnode); waitNodeState(ret, AdminStates.NORMAL); verifyStats(namenode, fsn, ret, false); @@ -509,7 +516,7 @@ public class TestDecommission { writeConfigFile(hostsFile, list); for (int j = 0; j < numNameNodes; j++) { - cluster.getNamesystem(j).refreshNodes(conf); + refreshNodes(cluster.getNamesystem(j), conf); DFSClient client = getDfsClient(cluster.getNameNode(j), conf); DatanodeInfo[] info = client.datanodeReport(DatanodeReportType.LIVE); diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestFileCorruption.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestFileCorruption.java index 5f673520c1c..8c0db5f2cc9 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestFileCorruption.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestFileCorruption.java @@ -146,8 +146,8 @@ public class TestFileCorruption extends TestCase { // report corrupted block by the third datanode DatanodeRegistration dnR = DataNodeTestUtils.getDNRegistrationForBP(dataNode, blk.getBlockPoolId()); - cluster.getNamesystem().markBlockAsCorrupt(blk, - new DatanodeInfo(dnR)); + cluster.getNamesystem().getBlockManager().findAndMarkBlockAsCorrupt( + blk, new DatanodeInfo(dnR)); // open the file fs.open(FILE_PATH); diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestFileCreation.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestFileCreation.java index 01cab9e4983..d2dfd7fc65e 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestFileCreation.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestFileCreation.java @@ -36,6 +36,7 @@ import org.apache.hadoop.fs.FsServerDefaults; import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; +import static org.apache.hadoop.hdfs.DFSConfigKeys.*; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; @@ -100,23 +101,23 @@ public class TestFileCreation extends junit.framework.TestCase { */ public void testServerDefaults() throws IOException { Configuration conf = new HdfsConfiguration(); - conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, FSConstants.DEFAULT_BLOCK_SIZE); - conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, FSConstants.DEFAULT_BYTES_PER_CHECKSUM); - conf.setInt(DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_KEY, FSConstants.DEFAULT_WRITE_PACKET_SIZE); - conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, FSConstants.DEFAULT_REPLICATION_FACTOR + 1); - conf.setInt("io.file.buffer.size", FSConstants.DEFAULT_FILE_BUFFER_SIZE); + conf.setLong(DFS_BLOCK_SIZE_KEY, DFS_BLOCK_SIZE_DEFAULT); + conf.setInt(DFS_BYTES_PER_CHECKSUM_KEY, DFS_BYTES_PER_CHECKSUM_DEFAULT); + conf.setInt(DFS_CLIENT_WRITE_PACKET_SIZE_KEY, DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT); + conf.setInt(DFS_REPLICATION_KEY, DFS_REPLICATION_DEFAULT + 1); + conf.setInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) - .numDataNodes(FSConstants.DEFAULT_REPLICATION_FACTOR + 1) + .numDataNodes(DFSConfigKeys.DFS_REPLICATION_DEFAULT + 1) .build(); cluster.waitActive(); FileSystem fs = cluster.getFileSystem(); try { FsServerDefaults serverDefaults = fs.getServerDefaults(); - assertEquals(FSConstants.DEFAULT_BLOCK_SIZE, serverDefaults.getBlockSize()); - assertEquals(FSConstants.DEFAULT_BYTES_PER_CHECKSUM, serverDefaults.getBytesPerChecksum()); - assertEquals(FSConstants.DEFAULT_WRITE_PACKET_SIZE, serverDefaults.getWritePacketSize()); - assertEquals(FSConstants.DEFAULT_REPLICATION_FACTOR + 1, serverDefaults.getReplication()); - assertEquals(FSConstants.DEFAULT_FILE_BUFFER_SIZE, serverDefaults.getFileBufferSize()); + assertEquals(DFS_BLOCK_SIZE_DEFAULT, serverDefaults.getBlockSize()); + assertEquals(DFS_BYTES_PER_CHECKSUM_DEFAULT, serverDefaults.getBytesPerChecksum()); + assertEquals(DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT, serverDefaults.getWritePacketSize()); + assertEquals(DFS_REPLICATION_DEFAULT + 1, serverDefaults.getReplication()); + assertEquals(IO_FILE_BUFFER_SIZE_DEFAULT, serverDefaults.getFileBufferSize()); } finally { fs.close(); cluster.shutdown(); @@ -269,8 +270,8 @@ public class TestFileCreation extends junit.framework.TestCase { */ public void testFileCreationError1() throws IOException { Configuration conf = new HdfsConfiguration(); - conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1000); - conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); + conf.setInt(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1000); + conf.setInt(DFS_HEARTBEAT_INTERVAL_KEY, 1); if (simulatedStorage) { conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); } @@ -343,8 +344,8 @@ public class TestFileCreation extends junit.framework.TestCase { long leasePeriod = 1000; System.out.println("testFileCreationError2 start"); Configuration conf = new HdfsConfiguration(); - conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1000); - conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); + conf.setInt(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1000); + conf.setInt(DFS_HEARTBEAT_INTERVAL_KEY, 1); if (simulatedStorage) { conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); } @@ -403,6 +404,36 @@ public class TestFileCreation extends junit.framework.TestCase { } } + /** test addBlock(..) when replicationbar/foo\">bar"), + }; + + @BeforeClass + public static void setUp() throws IOException { ((Log4JLogger)HftpFileSystem.LOG).getLogger().setLevel(Level.ALL); final long seed = RAN.nextLong(); @@ -67,66 +90,73 @@ public class TestHftpFileSystem extends TestCase { config = new Configuration(); config.set(DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY, "localhost"); - cluster = new MiniDFSCluster.Builder(config).numDataNodes(2).build(); hdfs = cluster.getFileSystem(); blockPoolId = cluster.getNamesystem().getBlockPoolId(); - final String hftpuri = + final String hftpUri = "hftp://" + config.get(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY); - hftpFs = (HftpFileSystem) new Path(hftpuri).getFileSystem(config); + hftpFs = (HftpFileSystem) new Path(hftpUri).getFileSystem(config); } - /** - * Shutdown the hadoop mini-cluster. - */ - private static void oneTimeTearDown() throws IOException { + @AfterClass + public static void tearDown() throws IOException { hdfs.close(); hftpFs.close(); cluster.shutdown(); } - - public TestHftpFileSystem(String name) { - super(name); - } /** - * For one time setup / teardown. + * Test file creation and access with file names that need encoding. */ - public static Test suite() { - TestSuite suite = new TestSuite(); - - suite.addTestSuite(TestHftpFileSystem.class); - - return new TestSetup(suite) { - @Override - protected void setUp() throws IOException { - oneTimeSetUp(); - } - - @Override - protected void tearDown() throws IOException { - oneTimeTearDown(); - } - }; - } - - public void testDataNodeRedirect() throws Exception { - if (hdfs.exists(TEST_FILE)) { - hdfs.delete(TEST_FILE, true); + @Test + public void testFileNameEncoding() throws IOException, URISyntaxException { + for (Path p : TEST_PATHS) { + // Create and access the path (data and streamFile servlets) + FSDataOutputStream out = hdfs.create(p, true); + out.writeBytes("0123456789"); + out.close(); + FSDataInputStream in = hftpFs.open(p); + assertEquals('0', in.read()); + + // Check the file status matches the path. Hftp returns a FileStatus + // with the entire URI, extract the path part. + assertEquals(p, new Path(hftpFs.getFileStatus(p).getPath().toUri().getPath())); + + // Test list status (listPath servlet) + assertEquals(1, hftpFs.listStatus(p).length); + + // Test content summary (contentSummary servlet) + assertNotNull("No content summary", hftpFs.getContentSummary(p)); + + // Test checksums (fileChecksum and getFileChecksum servlets) + assertNotNull("No file checksum", hftpFs.getFileChecksum(p)); } - FSDataOutputStream out = hdfs.create(TEST_FILE, (short) 1); + } + + private void testDataNodeRedirect(Path path) throws IOException { + // Create the file + if (hdfs.exists(path)) { + hdfs.delete(path, true); + } + FSDataOutputStream out = hdfs.create(path, (short)1); out.writeBytes("0123456789"); out.close(); - + + // Get the path's block location so we can determine + // if we were redirected to the right DN. BlockLocation[] locations = - hdfs.getFileBlockLocations(TEST_FILE, 0, 10); - + hdfs.getFileBlockLocations(path, 0, 10); String locationName = locations[0].getNames()[0]; - URL u = hftpFs.getNamenodeFileURL(TEST_FILE); + + // Connect to the NN to get redirected + URL u = hftpFs.getNamenodeURL( + "/data" + ServletUtil.encodePath(path.toUri().getPath()), + "ugi=userx,groupy"); HttpURLConnection conn = (HttpURLConnection)u.openConnection(); HttpURLConnection.setFollowRedirects(true); conn.connect(); conn.getInputStream(); + boolean checked = false; // Find the datanode that has the block according to locations // and check that the URL was redirected to this DN's info port @@ -138,19 +168,32 @@ public class TestHftpFileSystem extends TestCase { assertEquals(dnR.getInfoPort(), conn.getURL().getPort()); } } - assertTrue("The test never checked that location of " + - "the block and hftp desitnation are the same", checked); + assertTrue("The test never checked that location of " + + "the block and hftp desitnation are the same", checked); } + + /** + * Test that clients are redirected to the appropriate DN. + */ + @Test + public void testDataNodeRedirect() throws IOException { + for (Path p : TEST_PATHS) { + testDataNodeRedirect(p); + } + } + /** * Tests getPos() functionality. */ - public void testGetPos() throws Exception { + @Test + public void testGetPos() throws IOException { + final Path testFile = new Path("/testfile+1"); // Write a test file. - FSDataOutputStream out = hdfs.create(TEST_FILE, true); + FSDataOutputStream out = hdfs.create(testFile, true); out.writeBytes("0123456789"); out.close(); - FSDataInputStream in = hftpFs.open(TEST_FILE); + FSDataInputStream in = hftpFs.open(testFile); // Test read(). for (int i = 0; i < 5; ++i) { @@ -175,17 +218,17 @@ public class TestHftpFileSystem extends TestCase { assertEquals(10, in.getPos()); in.close(); } - + /** * Tests seek(). */ - public void testSeek() throws Exception { - // Write a test file. - FSDataOutputStream out = hdfs.create(TEST_FILE, true); + @Test + public void testSeek() throws IOException { + final Path testFile = new Path("/testfile+1"); + FSDataOutputStream out = hdfs.create(testFile, true); out.writeBytes("0123456789"); out.close(); - - FSDataInputStream in = hftpFs.open(TEST_FILE); + FSDataInputStream in = hftpFs.open(testFile); in.seek(7); assertEquals('7', in.read()); } diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestLease.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestLease.java index ca5658dc57f..a90c9d2ef3a 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestLease.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestLease.java @@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.ClientProtocol; +import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.security.UserGroupInformation; import org.junit.Assert; import org.junit.Test; @@ -31,7 +32,8 @@ import org.mockito.Mockito; public class TestLease { static boolean hasLease(MiniDFSCluster cluster, Path src) { - return cluster.getNamesystem().leaseManager.getLeaseByPath(src.toString()) != null; + return NameNodeAdapter.getLeaseManager(cluster.getNamesystem() + ).getLeaseByPath(src.toString()) != null; } final Path dir = new Path("/test/lease/"); diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestLeaseRecovery.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestLeaseRecovery.java index 6e65ad2c51d..3e084e15476 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestLeaseRecovery.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestLeaseRecovery.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.TestInterDatanodeProtocol; import org.apache.hadoop.hdfs.server.namenode.LeaseManager; +import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; public class TestLeaseRecovery extends junit.framework.TestCase { static final int BLOCK_SIZE = 1024; @@ -133,7 +134,7 @@ public class TestLeaseRecovery extends junit.framework.TestCase { DFSTestUtil.waitReplication(dfs, filepath, (short)1); waitLeaseRecovery(cluster); // verify that we still cannot recover the lease - LeaseManager lm = cluster.getNamesystem().leaseManager; + LeaseManager lm = NameNodeAdapter.getLeaseManager(cluster.getNamesystem()); assertTrue("Found " + lm.countLease() + " lease, expected 1", lm.countLease() == 1); cluster.getNameNode().setSafeMode(FSConstants.SafeModeAction.SAFEMODE_LEAVE); } diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java index 9ab797afdb3..26f42631fd8 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java @@ -29,6 +29,7 @@ import org.apache.hadoop.fs.ChecksumException; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; /** * The test makes sure that NameNode detects presense blocks that do not have @@ -56,6 +57,7 @@ public class TestMissingBlocksAlert extends TestCase { cluster = new MiniDFSCluster.Builder(conf).build(); cluster.waitActive(); + final BlockManager bm = cluster.getNamesystem().getBlockManager(); DistributedFileSystem dfs = (DistributedFileSystem) cluster.getFileSystem(); @@ -86,8 +88,7 @@ public class TestMissingBlocksAlert extends TestCase { } assertTrue(dfs.getMissingBlocksCount() == 1); assertEquals(4, dfs.getUnderReplicatedBlocksCount()); - assertEquals(3, - cluster.getNamesystem().getUnderReplicatedNotMissingBlocks()); + assertEquals(3, bm.getUnderReplicatedNotMissingBlocks()); // Now verify that it shows up on webui @@ -109,8 +110,7 @@ public class TestMissingBlocksAlert extends TestCase { } assertEquals(2, dfs.getUnderReplicatedBlocksCount()); - assertEquals(2, - cluster.getNamesystem().getUnderReplicatedNotMissingBlocks()); + assertEquals(2, bm.getUnderReplicatedNotMissingBlocks()); // and make sure WARNING disappears // Now verify that it shows up on webui diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/security/TestDelegationToken.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/security/TestDelegationToken.java index e109ae09677..d6397b6a2ee 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/security/TestDelegationToken.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/security/TestDelegationToken.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager; +import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; @@ -48,7 +49,8 @@ import org.junit.Test; public class TestDelegationToken { private MiniDFSCluster cluster; - Configuration config; + private DelegationTokenSecretManager dtSecretManager; + private Configuration config; private static final Log LOG = LogFactory.getLog(TestDelegationToken.class); @Before @@ -61,7 +63,9 @@ public class TestDelegationToken { FileSystem.setDefaultUri(config, "hdfs://localhost:" + "0"); cluster = new MiniDFSCluster.Builder(config).build(); cluster.waitActive(); - cluster.getNamesystem().getDelegationTokenSecretManager().startThreads(); + dtSecretManager = NameNodeAdapter.getDtSecretManager( + cluster.getNamesystem()); + dtSecretManager.startThreads(); } @After @@ -73,8 +77,6 @@ public class TestDelegationToken { private Token generateDelegationToken( String owner, String renewer) { - DelegationTokenSecretManager dtSecretManager = cluster.getNamesystem() - .getDelegationTokenSecretManager(); DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(new Text( owner), new Text(renewer), null); return new Token(dtId, dtSecretManager); @@ -82,8 +84,6 @@ public class TestDelegationToken { @Test public void testDelegationTokenSecretManager() throws Exception { - DelegationTokenSecretManager dtSecretManager = cluster.getNamesystem() - .getDelegationTokenSecretManager(); Token token = generateDelegationToken( "SomeUser", "JobTracker"); // Fake renewer should not be able to renew @@ -122,8 +122,6 @@ public class TestDelegationToken { @Test public void testCancelDelegationToken() throws Exception { - DelegationTokenSecretManager dtSecretManager = cluster.getNamesystem() - .getDelegationTokenSecretManager(); Token token = generateDelegationToken( "SomeUser", "JobTracker"); //Fake renewer should not be able to renew @@ -144,7 +142,6 @@ public class TestDelegationToken { @Test public void testDelegationTokenDFSApi() throws Exception { - DelegationTokenSecretManager dtSecretManager = cluster.getNamesystem().getDelegationTokenSecretManager(); DistributedFileSystem dfs = (DistributedFileSystem) cluster.getFileSystem(); Token token = dfs.getDelegationToken("JobTracker"); DelegationTokenIdentifier identifier = new DelegationTokenIdentifier(); diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java index c9135287cdc..4c5e1f9c0cd 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java @@ -31,19 +31,20 @@ import java.util.Enumeration; import junit.framework.Assert; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.io.Text; -import org.apache.commons.logging.*; +import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; +import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.security.TestDoAsEffectiveUser; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -98,7 +99,7 @@ public class TestDelegationTokenForProxyUser { FileSystem.setDefaultUri(config, "hdfs://localhost:" + "0"); cluster = new MiniDFSCluster.Builder(config).build(); cluster.waitActive(); - cluster.getNamesystem().getDelegationTokenSecretManager().startThreads(); + NameNodeAdapter.getDtSecretManager(cluster.getNamesystem()).startThreads(); ProxyUsers.refreshSuperUserGroupsConfiguration(config); } diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java index 292f7d817cc..2d2406b12b9 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java @@ -25,10 +25,13 @@ import java.util.Set; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; -import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.util.Daemon; public class BlockManagerTestUtil { + public static void setNodeReplicationLimit(final BlockManager blockManager, + final int limit) { + blockManager.maxReplicationStreams = limit; + } /** @return the datanode descriptor for the given the given storageID. */ public static DatanodeDescriptor getDatanode(final FSNamesystem ns, diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBlocksWithNotEnoughRacks.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java similarity index 95% rename from hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBlocksWithNotEnoughRacks.java rename to hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java index 95d1ecafcee..01018e8c05a 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBlocksWithNotEnoughRacks.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java @@ -16,10 +16,16 @@ * limitations under the License. */ -package org.apache.hadoop.hdfs.server.namenode; +package org.apache.hadoop.hdfs.server.blockmanagement; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import java.util.ArrayList; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FileSystem; @@ -31,18 +37,15 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.server.datanode.DataNode; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.commons.logging.impl.Log4JLogger; +import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.log4j.Level; - -import static org.junit.Assert.*; import org.junit.Test; public class TestBlocksWithNotEnoughRacks { public static final Log LOG = LogFactory.getLog(TestBlocksWithNotEnoughRacks.class); static { - ((Log4JLogger)FSNamesystem.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.ALL); ((Log4JLogger)LOG).getLogger().setLevel(Level.ALL); } @@ -131,7 +134,7 @@ public class TestBlocksWithNotEnoughRacks { DFSTestUtil.waitForReplication(cluster, b, 1, REPLICATION_FACTOR, 0); REPLICATION_FACTOR = 2; - ns.setReplication("/testFile", REPLICATION_FACTOR); + NameNodeAdapter.setReplication(ns, "/testFile", REPLICATION_FACTOR); DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0); } finally { cluster.shutdown(); @@ -170,7 +173,7 @@ public class TestBlocksWithNotEnoughRacks { String newRacks[] = {"/rack2", "/rack2"}; cluster.startDataNodes(conf, 2, true, null, newRacks); REPLICATION_FACTOR = 5; - ns.setReplication("/testFile", REPLICATION_FACTOR); + NameNodeAdapter.setReplication(ns, "/testFile", REPLICATION_FACTOR); DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0); } finally { @@ -256,7 +259,7 @@ public class TestBlocksWithNotEnoughRacks { // was not the one that lived on the rack with only one replica, // ie we should still have 2 racks after reducing the repl factor. REPLICATION_FACTOR = 2; - ns.setReplication("/testFile", REPLICATION_FACTOR); + NameNodeAdapter.setReplication(ns, "/testFile", REPLICATION_FACTOR); DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0); } finally { @@ -278,6 +281,7 @@ public class TestBlocksWithNotEnoughRacks { MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(racks.length).racks(racks).build(); final FSNamesystem ns = cluster.getNameNode().getNamesystem(); + final DatanodeManager dm = ns.getBlockManager().getDatanodeManager(); try { // Create a file with one block with a replication factor of 2 @@ -293,7 +297,7 @@ public class TestBlocksWithNotEnoughRacks { DataNode dataNode = datanodes.get(idx); DatanodeID dnId = dataNode.getDatanodeId(); cluster.stopDataNode(idx); - ns.removeDatanode(dnId); + dm.removeDatanode(dnId); // The block should still have sufficient # replicas, across racks. // The last node may not have contained a replica, but if it did @@ -307,7 +311,7 @@ public class TestBlocksWithNotEnoughRacks { dataNode = datanodes.get(idx); dnId = dataNode.getDatanodeId(); cluster.stopDataNode(idx); - ns.removeDatanode(dnId); + dm.removeDatanode(dnId); // Make sure we have enough live replicas even though we are // short one rack and therefore need one replica @@ -332,6 +336,7 @@ public class TestBlocksWithNotEnoughRacks { MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(racks.length).racks(racks).build(); final FSNamesystem ns = cluster.getNameNode().getNamesystem(); + final DatanodeManager dm = ns.getBlockManager().getDatanodeManager(); try { // Create a file with one block @@ -347,7 +352,7 @@ public class TestBlocksWithNotEnoughRacks { DataNode dataNode = datanodes.get(2); DatanodeID dnId = dataNode.getDatanodeId(); cluster.stopDataNode(2); - ns.removeDatanode(dnId); + dm.removeDatanode(dnId); // The block gets re-replicated to another datanode so it has a // sufficient # replicas, but not across racks, so there should @@ -408,7 +413,7 @@ public class TestBlocksWithNotEnoughRacks { fs.getFileStatus(filePath), 0, Long.MAX_VALUE); String name = locs[0].getNames()[0]; DFSTestUtil.writeFile(localFileSys, excludeFile, name); - ns.refreshNodes(conf); + ns.getBlockManager().getDatanodeManager().refreshNodes(conf); DFSTestUtil.waitForDecommission(fs, name); // Check the block still has sufficient # replicas across racks @@ -463,7 +468,7 @@ public class TestBlocksWithNotEnoughRacks { if (!top.startsWith("/rack2")) { String name = top.substring("/rack1".length()+1); DFSTestUtil.writeFile(localFileSys, excludeFile, name); - ns.refreshNodes(conf); + ns.getBlockManager().getDatanodeManager().refreshNodes(conf); DFSTestUtil.waitForDecommission(fs, name); break; } diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java index 3e32a95cb0d..2d676578ffe 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java @@ -45,8 +45,8 @@ public class TestComputeInvalidateWork extends TestCase { final FSNamesystem namesystem = cluster.getNamesystem(); final BlockManager bm = namesystem.getBlockManager(); final int blockInvalidateLimit = bm.getDatanodeManager().blockInvalidateLimit; - DatanodeDescriptor[] nodes = - namesystem.heartbeats.toArray(new DatanodeDescriptor[NUM_OF_DATANODES]); + final DatanodeDescriptor[] nodes = bm.getDatanodeManager( + ).getHeartbeatManager().getDatanodes(); assertEquals(nodes.length, NUM_OF_DATANODES); namesystem.writeLock(); diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java index 9b2b322cded..c18a5c04fe0 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import java.io.IOException; import java.util.ArrayList; import junit.framework.TestCase; @@ -52,6 +51,8 @@ public class TestHeartbeatHandling extends TestCase { try { cluster.waitActive(); final FSNamesystem namesystem = cluster.getNamesystem(); + final HeartbeatManager hm = namesystem.getBlockManager( + ).getDatanodeManager().getHeartbeatManager(); final String poolId = namesystem.getBlockPoolId(); final DatanodeRegistration nodeReg = DataNodeTestUtils.getDNRegistrationForBP(cluster.getDataNodes().get(0), poolId); @@ -69,12 +70,12 @@ public class TestHeartbeatHandling extends TestCase { try { namesystem.writeLock(); - synchronized (namesystem.heartbeats) { + synchronized(hm) { for (int i=0; i iter = namesystem.getBlockManager().blocksMap + final Iterator iter = bm.blocksMap .nodeIterator(block.getLocalBlock()); DatanodeDescriptor nonExcessDN = null; while (iter.hasNext()) { DatanodeDescriptor dn = iter.next(); - Collection blocks = namesystem.getBlockManager().excessReplicateMap.get(dn.getStorageID()); + Collection blocks = bm.excessReplicateMap.get(dn.getStorageID()); if (blocks == null || !blocks.contains(block) ) { nonExcessDN = dn; break; @@ -121,9 +124,9 @@ public class TestNodeCount extends TestCase { try { namesystem.writeLock(); - synchronized (namesystem.heartbeats) { + synchronized(hm) { nonExcessDN.setLastUpdate(0); // mark it dead - namesystem.heartbeatCheck(); + hm.heartbeatCheck(); } } finally { namesystem.writeUnlock(); diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestOverReplicatedBlocks.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java similarity index 88% rename from hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestOverReplicatedBlocks.java rename to hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java index c4b91c21c33..102b41ca8e0 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestOverReplicatedBlocks.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hdfs.server.namenode; +package org.apache.hadoop.hdfs.server.blockmanagement; import java.io.File; import java.io.IOException; @@ -33,8 +33,9 @@ import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties; import org.apache.hadoop.hdfs.TestDatanodeBlockScanner; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; -import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; +import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; public class TestOverReplicatedBlocks extends TestCase { /** Test processOverReplicatedBlock can handle corrupt replicas fine. @@ -83,25 +84,26 @@ public class TestOverReplicatedBlocks extends TestCase { cluster.getDataNodes().get(2), blockPoolId); final FSNamesystem namesystem = cluster.getNamesystem(); + final BlockManager bm = namesystem.getBlockManager(); + final HeartbeatManager hm = bm.getDatanodeManager().getHeartbeatManager(); try { namesystem.writeLock(); - synchronized (namesystem.heartbeats) { + synchronized(hm) { // set live datanode's remaining space to be 0 // so they will be chosen to be deleted when over-replication occurs String corruptMachineName = corruptDataNode.getName(); - for (DatanodeDescriptor datanode : namesystem.heartbeats) { + for (DatanodeDescriptor datanode : hm.getDatanodes()) { if (!corruptMachineName.equals(datanode.getName())) { datanode.updateHeartbeat(100L, 100L, 0L, 100L, 0, 0); } } // decrease the replication factor to 1; - namesystem.setReplication(fileName.toString(), (short)1); + NameNodeAdapter.setReplication(namesystem, fileName.toString(), (short)1); // corrupt one won't be chosen to be excess one // without 4910 the number of live replicas would be 0: block gets lost - assertEquals(1, namesystem.getBlockManager().countNodes(block.getLocalBlock()) - .liveReplicas()); + assertEquals(1, bm.countNodes(block.getLocalBlock()).liveReplicas()); } } finally { namesystem.writeUnlock(); diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index 9e711964b4d..d2cfe699b64 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -63,7 +63,7 @@ import org.apache.hadoop.util.DiskChecker.DiskErrorException; * Note the synchronization is coarse grained - it is at each method. */ -public class SimulatedFSDataset implements FSConstants, FSDatasetInterface, Configurable{ +public class SimulatedFSDataset implements FSDatasetInterface, Configurable{ public static final String CONFIG_PROPERTY_SIMULATED = "dfs.datanode.simulateddatastorage"; diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java index 094673b3237..e53933539e4 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java @@ -37,7 +37,7 @@ import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; -import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; import org.apache.log4j.Level; import org.junit.After; import org.junit.Before; @@ -113,10 +113,11 @@ public class TestDataNodeVolumeFailureReporting { * heartbeat their capacities. */ Thread.sleep(WAIT_FOR_HEARTBEATS); - FSNamesystem ns = cluster.getNamesystem(); + final DatanodeManager dm = cluster.getNamesystem().getBlockManager( + ).getDatanodeManager(); - long origCapacity = DFSTestUtil.getLiveDatanodeCapacity(ns); - long dnCapacity = DFSTestUtil.getDatanodeCapacity(ns, 0); + final long origCapacity = DFSTestUtil.getLiveDatanodeCapacity(dm); + long dnCapacity = DFSTestUtil.getDatanodeCapacity(dm, 0); File dn1Vol1 = new File(dataDir, "data"+(2*0+1)); File dn2Vol1 = new File(dataDir, "data"+(2*1+1)); @@ -160,7 +161,7 @@ public class TestDataNodeVolumeFailureReporting { assert (WAIT_FOR_HEARTBEATS * 10) > WAIT_FOR_DEATH; // Eventually the NN should report two volume failures - DFSTestUtil.waitForDatanodeStatus(ns, 3, 0, 2, + DFSTestUtil.waitForDatanodeStatus(dm, 3, 0, 2, origCapacity - (1*dnCapacity), WAIT_FOR_HEARTBEATS); /* @@ -177,10 +178,10 @@ public class TestDataNodeVolumeFailureReporting { ArrayList live = new ArrayList(); ArrayList dead = new ArrayList(); - ns.DFSNodesStatus(live, dead); + dm.fetchDatanodes(live, dead, false); live.clear(); dead.clear(); - ns.DFSNodesStatus(live, dead); + dm.fetchDatanodes(live, dead, false); assertEquals("DN3 should have 1 failed volume", 1, live.get(2).getVolumeFailures()); @@ -189,8 +190,8 @@ public class TestDataNodeVolumeFailureReporting { * total capacity should be down by three volumes (assuming the host * did not grow or shrink the data volume while the test was running). */ - dnCapacity = DFSTestUtil.getDatanodeCapacity(ns, 0); - DFSTestUtil.waitForDatanodeStatus(ns, 3, 0, 3, + dnCapacity = DFSTestUtil.getDatanodeCapacity(dm, 0); + DFSTestUtil.waitForDatanodeStatus(dm, 3, 0, 3, origCapacity - (3*dnCapacity), WAIT_FOR_HEARTBEATS); /* @@ -212,7 +213,7 @@ public class TestDataNodeVolumeFailureReporting { getMetrics(dns.get(2).getMetrics().name())); // The NN considers the DN dead - DFSTestUtil.waitForDatanodeStatus(ns, 2, 1, 2, + DFSTestUtil.waitForDatanodeStatus(dm, 2, 1, 2, origCapacity - (4*dnCapacity), WAIT_FOR_HEARTBEATS); /* @@ -236,7 +237,7 @@ public class TestDataNodeVolumeFailureReporting { * and that the volume failure count should be reported as zero by * both the metrics and the NN. */ - DFSTestUtil.waitForDatanodeStatus(ns, 3, 0, 0, origCapacity, + DFSTestUtil.waitForDatanodeStatus(dm, 3, 0, 0, origCapacity, WAIT_FOR_HEARTBEATS); } @@ -251,9 +252,10 @@ public class TestDataNodeVolumeFailureReporting { cluster.startDataNodes(conf, 2, true, null, null); cluster.waitActive(); - FSNamesystem ns = cluster.getNamesystem(); - long origCapacity = DFSTestUtil.getLiveDatanodeCapacity(ns); - long dnCapacity = DFSTestUtil.getDatanodeCapacity(ns, 0); + final DatanodeManager dm = cluster.getNamesystem().getBlockManager( + ).getDatanodeManager(); + long origCapacity = DFSTestUtil.getLiveDatanodeCapacity(dm); + long dnCapacity = DFSTestUtil.getDatanodeCapacity(dm, 0); // Fail the first volume on both datanodes (we have to keep the // third healthy so one node in the pipeline will not fail). @@ -267,13 +269,13 @@ public class TestDataNodeVolumeFailureReporting { DFSTestUtil.waitReplication(fs, file1, (short)2); // The NN reports two volumes failures - DFSTestUtil.waitForDatanodeStatus(ns, 3, 0, 2, + DFSTestUtil.waitForDatanodeStatus(dm, 3, 0, 2, origCapacity - (1*dnCapacity), WAIT_FOR_HEARTBEATS); // After restarting the NN it still see the two failures cluster.restartNameNode(0); cluster.waitActive(); - DFSTestUtil.waitForDatanodeStatus(ns, 3, 0, 2, + DFSTestUtil.waitForDatanodeStatus(dm, 3, 0, 2, origCapacity - (1*dnCapacity), WAIT_FOR_HEARTBEATS); } } diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java index d5f65c256e1..5303799e9d5 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java @@ -17,29 +17,32 @@ */ package org.apache.hadoop.hdfs.server.datanode; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + import java.io.File; import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DFSTestUtil; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; -import org.apache.hadoop.hdfs.HdfsConfiguration; -import org.apache.hadoop.hdfs.DFSConfigKeys; +import java.util.concurrent.TimeoutException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.impl.Log4JLogger; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; +import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.log4j.Level; - import org.junit.After; import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.*; -import static org.junit.Assume.assumeTrue; /** * Test the ability of a DN to tolerate volume failures. @@ -154,9 +157,10 @@ public class TestDataNodeVolumeFailureToleration { conf.setInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY, 0); cluster.startDataNodes(conf, 2, true, null, null); cluster.waitActive(); - FSNamesystem ns = cluster.getNamesystem(); - long origCapacity = DFSTestUtil.getLiveDatanodeCapacity(ns); - long dnCapacity = DFSTestUtil.getDatanodeCapacity(ns, 0); + final DatanodeManager dm = cluster.getNamesystem().getBlockManager( + ).getDatanodeManager(); + long origCapacity = DFSTestUtil.getLiveDatanodeCapacity(dm); + long dnCapacity = DFSTestUtil.getDatanodeCapacity(dm, 0); // Fail a volume on the 2nd DN File dn2Vol1 = new File(dataDir, "data"+(2*1+1)); @@ -168,7 +172,7 @@ public class TestDataNodeVolumeFailureToleration { DFSTestUtil.waitReplication(fs, file1, (short)2); // Check that this single failure caused a DN to die. - DFSTestUtil.waitForDatanodeStatus(ns, 2, 1, 0, + DFSTestUtil.waitForDatanodeStatus(dm, 2, 1, 0, origCapacity - (1*dnCapacity), WAIT_FOR_HEARTBEATS); // If we restore the volume we should still only be able to get @@ -187,7 +191,7 @@ public class TestDataNodeVolumeFailureToleration { */ private void restartDatanodes(int volTolerated, boolean manageDfsDirs) throws IOException { - //Make sure no datanode is running + // Make sure no datanode is running cluster.shutdownDataNodes(); conf.setInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY, volTolerated); cluster.startDataNodes(conf, 1, manageDfsDirs, null, null); @@ -224,7 +228,7 @@ public class TestDataNodeVolumeFailureToleration { */ private void testVolumeConfig(int volumesTolerated, int volumesFailed, boolean expectedBPServiceState, boolean manageDfsDirs) - throws IOException, InterruptedException { + throws IOException, InterruptedException, TimeoutException { assumeTrue(!System.getProperty("os.name").startsWith("Windows")); final int dnIndex = 0; // Fail the current directory since invalid storage directory perms @@ -259,4 +263,30 @@ public class TestDataNodeVolumeFailureToleration { assertEquals("Couldn't chmod local vol", 0, FileUtil.chmod(dir.toString(), "000")); } + + /** + * Test that a volume that is considered failed on startup is seen as + * a failed volume by the NN. + */ + @Test + public void testFailedVolumeOnStartupIsCounted() throws Exception { + assumeTrue(!System.getProperty("os.name").startsWith("Windows")); + final DatanodeManager dm = cluster.getNamesystem().getBlockManager( + ).getDatanodeManager(); + long origCapacity = DFSTestUtil.getLiveDatanodeCapacity(dm); + File dir = new File(MiniDFSCluster.getStorageDir(0, 0), "current"); + + try { + prepareDirToFail(dir); + restartDatanodes(1, false); + // The cluster is up.. + assertEquals(true, cluster.getDataNodes().get(0) + .isBPServiceAlive(cluster.getNamesystem().getBlockPoolId())); + // but there has been a single volume failure + DFSTestUtil.waitForDatanodeStatus(dm, 1, 0, 1, + origCapacity / 2, WAIT_FOR_HEARTBEATS); + } finally { + FileUtil.chmod(dir.toString(), "755"); + } + } } diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDatanodeJsp.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDatanodeJsp.java index 1eaf5d0574e..eb78aced85a 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDatanodeJsp.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDatanodeJsp.java @@ -28,7 +28,6 @@ import java.net.URLEncoder; import javax.servlet.http.HttpServletRequest; import javax.servlet.jsp.JspWriter; -import org.apache.commons.httpclient.util.URIUtil; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSTestUtil; @@ -36,6 +35,7 @@ import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.util.ServletUtil; import org.junit.Test; import org.mockito.Mockito; @@ -71,7 +71,7 @@ public class TestDatanodeJsp { if (!doTail) { assertTrue("page should show link to download file", viewFilePage - .contains("/streamFile" + URIUtil.encodePath(testPath.toString()) + + .contains("/streamFile" + ServletUtil.encodePath(testPath.toString()) + "?nnaddr=localhost:" + nnIpcAddress.getPort())); } } @@ -82,15 +82,22 @@ public class TestDatanodeJsp { try { cluster = new MiniDFSCluster.Builder(CONF).build(); cluster.waitActive(); - - testViewingFile(cluster, "/test-file", false); - testViewingFile(cluster, "/tmp/test-file", false); - testViewingFile(cluster, "/tmp/test-file%with goofy&characters", false); - - testViewingFile(cluster, "/test-file", true); - testViewingFile(cluster, "/tmp/test-file", true); - testViewingFile(cluster, "/tmp/test-file%with goofy&characters", true); - + String paths[] = { + "/test-file", + "/tmp/test-file", + "/tmp/test-file%with goofy&characters", + "/foo bar/foo bar", + "/foo+bar/foo+bar", + "/foo;bar/foo;bar", + "/foo=bar/foo=bar", + "/foo,bar/foo,bar", + "/foo?bar/foo?bar", + "/foo\">bar/foo\">bar" + }; + for (String p : paths) { + testViewingFile(cluster, p, false); + testViewingFile(cluster, p, true); + } } finally { if (cluster != null) { cluster.shutdown(); diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java index 39d64f0686f..5deccd5c221 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java @@ -24,25 +24,27 @@ import java.io.IOException; import java.io.RandomAccessFile; import java.net.URI; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; +import org.apache.commons.logging.Log; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.server.common.Storage.StorageDirType; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; -import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundEditLog; -import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundFSImage; +import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile; +import org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.FSImageFile; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; import org.apache.hadoop.hdfs.util.MD5FileUtils; import org.apache.hadoop.io.IOUtils; import org.mockito.Mockito; import com.google.common.base.Joiner; -import com.google.common.collect.ComparisonChain; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -110,6 +112,40 @@ public abstract class FSImageTestUtil { return sd; } + /** + * Make a mock storage directory that returns some set of file contents. + * @param type type of storage dir + * @param previousExists should we mock that the previous/ dir exists? + * @param fileNames the names of files contained in current/ + */ + static StorageDirectory mockStorageDirectory( + StorageDirType type, + boolean previousExists, + String... fileNames) { + StorageDirectory sd = mock(StorageDirectory.class); + + doReturn(type).when(sd).getStorageDirType(); + + // Version file should always exist + doReturn(mockFile(true)).when(sd).getVersionFile(); + + // Previous dir optionally exists + doReturn(mockFile(previousExists)) + .when(sd).getPreviousDir(); + + // Return a mock 'current' directory which has the given paths + File[] files = new File[fileNames.length]; + for (int i = 0; i < fileNames.length; i++) { + files[i] = new File(fileNames[i]); + } + + File mockDir = Mockito.spy(new File("/dir/current")); + doReturn(files).when(mockDir).listFiles(); + doReturn(mockDir).when(sd).getCurrentDir(); + + return sd; + } + static File mockFile(boolean exists) { File mockFile = mock(File.class); doReturn(exists).when(mockFile).exists(); @@ -154,9 +190,9 @@ public abstract class FSImageTestUtil { for (File dir : dirs) { FSImageTransactionalStorageInspector inspector = inspectStorageDirectory(dir, NameNodeDirType.IMAGE); - FoundFSImage latestImage = inspector.getLatestImage(); + FSImageFile latestImage = inspector.getLatestImage(); assertNotNull("No image in " + dir, latestImage); - long thisTxId = latestImage.getTxId(); + long thisTxId = latestImage.getCheckpointTxId(); if (imageTxId != -1 && thisTxId != imageTxId) { fail("Storage directory " + dir + " does not have the same " + "last image index " + imageTxId + " as another"); @@ -283,7 +319,7 @@ public abstract class FSImageTestUtil { new FSImageTransactionalStorageInspector(); inspector.inspectDirectory(sd); - FoundFSImage latestImage = inspector.getLatestImage(); + FSImageFile latestImage = inspector.getLatestImage(); return (latestImage == null) ? null : latestImage.getFile(); } @@ -316,23 +352,15 @@ public abstract class FSImageTestUtil { * @return the latest edits log, finalized or otherwise, from the given * storage directory. */ - public static FoundEditLog findLatestEditsLog(StorageDirectory sd) + public static EditLogFile findLatestEditsLog(StorageDirectory sd) throws IOException { FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector(); inspector.inspectDirectory(sd); - List foundEditLogs = Lists.newArrayList( - inspector.getFoundEditLogs()); - return Collections.max(foundEditLogs, new Comparator() { - @Override - public int compare(FoundEditLog a, FoundEditLog b) { - return ComparisonChain.start() - .compare(a.getStartTxId(), b.getStartTxId()) - .compare(a.getLastTxId(), b.getLastTxId()) - .result(); - } - }); + List foundEditLogs = Lists.newArrayList( + inspector.getEditLogFiles()); + return Collections.max(foundEditLogs, EditLogFile.COMPARE_BY_START_TXID); } /** @@ -371,5 +399,16 @@ public abstract class FSImageTestUtil { assertNotNull(image); } - + public static void logStorageContents(Log LOG, NNStorage storage) { + LOG.info("current storages and corresponding sizes:"); + for (StorageDirectory sd : storage.dirIterable(null)) { + File curDir = sd.getCurrentDir(); + LOG.info("In directory " + curDir); + File[] files = curDir.listFiles(); + Arrays.sort(files); + for (File f : files) { + LOG.info(" file " + f.getAbsolutePath() + "; len = " + f.length()); + } + } + } } diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java index d90ee5eac1f..ea3c30c28e4 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java @@ -1128,7 +1128,8 @@ public class NNThroughputBenchmark { // decommission data-nodes decommissionNodes(); // set node replication limit - namesystem.setNodeReplicationLimit(nodeReplicationLimit); + BlockManagerTestUtil.setNodeReplicationLimit(namesystem.getBlockManager(), + nodeReplicationLimit); } private void decommissionNodes() throws IOException { @@ -1171,9 +1172,7 @@ public class NNThroughputBenchmark { void printResults() { String blockDistribution = ""; String delim = "("; - int totalReplicas = 0; for(int idx=0; idx < blockReportObject.getNumDatanodes(); idx++) { - totalReplicas += blockReportObject.datanodes[idx].nrBlocks; blockDistribution += delim + blockReportObject.datanodes[idx].nrBlocks; delim = ", "; } diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java index 919dd718c50..afa39dfcf20 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java @@ -19,10 +19,12 @@ package org.apache.hadoop.hdfs.server.namenode; import java.io.IOException; -import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; +import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; +import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand; +import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.ipc.Server; /** @@ -52,7 +54,33 @@ public class NameNodeAdapter { public static Server getRpcServer(NameNode namenode) { return namenode.server; } + + public static DelegationTokenSecretManager getDtSecretManager( + final FSNamesystem ns) { + return ns.getDelegationTokenSecretManager(); + } + + public static DatanodeCommand[] sendHeartBeat(DatanodeRegistration nodeReg, + DatanodeDescriptor dd, FSNamesystem namesystem) throws IOException { + return namesystem.handleHeartbeat(nodeReg, dd.getCapacity(), + dd.getDfsUsed(), dd.getRemaining(), dd.getBlockPoolUsed(), 0, 0, 0); + } + + public static boolean setReplication(final FSNamesystem ns, + final String src, final short replication) throws IOException { + return ns.setReplication(src, replication); + } + public static LeaseManager getLeaseManager(final FSNamesystem ns) { + return ns.leaseManager; + } + + /** Set the softLimit and hardLimit of client lease periods. */ + public static void setLeasePeriod(final FSNamesystem namesystem, long soft, long hard) { + getLeaseManager(namesystem).setLeasePeriod(soft, hard); + namesystem.lmthread.interrupt(); + } + public static String getLeaseHolderForPath(NameNode namenode, String path) { return namenode.getNamesystem().leaseManager.getLeaseByPath(path).getHolder(); } diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java index c869adc9fac..72e3bf72308 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java @@ -33,7 +33,7 @@ import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; -import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundEditLog; +import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.test.GenericTestUtils; import org.apache.log4j.Level; @@ -163,8 +163,8 @@ public class TestBackupNode extends TestCase { // When shutting down the BN, it shouldn't finalize logs that are // still open on the NN - FoundEditLog editsLog = FSImageTestUtil.findLatestEditsLog(sd); - assertEquals(editsLog.getStartTxId(), + EditLogFile editsLog = FSImageTestUtil.findLatestEditsLog(sd); + assertEquals(editsLog.getFirstTxId(), nn.getFSImage().getEditLog().getCurSegmentTxId()); assertTrue("Should not have finalized " + editsLog, editsLog.isInProgress()); diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java index 44efa548e83..4f698c08d30 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java @@ -28,7 +28,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; -import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundEditLog; +import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile; import org.apache.hadoop.hdfs.tools.DFSAdmin; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.UserGroupInformation; @@ -82,7 +82,7 @@ public class TestCheckPointForSecurityTokens { // verify that the edits file is NOT empty NameNode nn = cluster.getNameNode(); for (StorageDirectory sd : nn.getFSImage().getStorage().dirIterable(null)) { - FoundEditLog log = FSImageTestUtil.findLatestEditsLog(sd); + EditLogFile log = FSImageTestUtil.findLatestEditsLog(sd); assertTrue(log.isInProgress()); assertEquals("In-progress log " + log + " should have 5 transactions", 5, log.validateLog().numTransactions); @@ -97,7 +97,7 @@ public class TestCheckPointForSecurityTokens { } // verify that the edits file is empty except for the START txn for (StorageDirectory sd : nn.getFSImage().getStorage().dirIterable(null)) { - FoundEditLog log = FSImageTestUtil.findLatestEditsLog(sd); + EditLogFile log = FSImageTestUtil.findLatestEditsLog(sd); assertTrue(log.isInProgress()); assertEquals("In-progress log " + log + " should only have START txn", 1, log.validateLog().numTransactions); diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java index 9855af88920..96d46783fec 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java @@ -1450,7 +1450,7 @@ public class TestCheckpoint extends TestCase { // Make a finalized log on the server side. nn.rollEditLog(); - RemoteEditLogManifest manifest = nn.getEditLogManifest(0); + RemoteEditLogManifest manifest = nn.getEditLogManifest(1); RemoteEditLog log = manifest.getLogs().get(0); NNStorage dstImage = Mockito.mock(NNStorage.class); diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java index 78dfb738720..be78c0df3a6 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Iterator; +import java.util.List; import java.util.Random; import org.apache.hadoop.conf.Configuration; @@ -37,6 +38,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.FSConstants.DatanodeReportType; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -146,7 +148,7 @@ public class TestDecommissioningStatus { /* * Decommissions the node at the given index */ - private String decommissionNode(FSNamesystem namesystem, Configuration conf, + private String decommissionNode(FSNamesystem namesystem, DFSClient client, FileSystem localFileSys, int nodeIndex) throws IOException { DatanodeInfo[] info = client.datanodeReport(DatanodeReportType.LIVE); @@ -158,7 +160,6 @@ public class TestDecommissioningStatus { ArrayList nodes = new ArrayList(decommissionedNodes); nodes.add(nodename); writeConfigFile(localFileSys, excludeFile, nodes); - namesystem.refreshNodes(conf); return nodename; } @@ -199,13 +200,13 @@ public class TestDecommissioningStatus { Thread.sleep(5000); FSNamesystem fsn = cluster.getNamesystem(); + final DatanodeManager dm = fsn.getBlockManager().getDatanodeManager(); for (int iteration = 0; iteration < numDatanodes; iteration++) { - String downnode = decommissionNode(fsn, conf, client, localFileSys, - iteration); + String downnode = decommissionNode(fsn, client, localFileSys, iteration); + dm.refreshNodes(conf); decommissionedNodes.add(downnode); Thread.sleep(5000); - ArrayList decommissioningNodes = fsn - .getDecommissioningNodes(); + final List decommissioningNodes = dm.getDecommissioningNodes(); if (iteration == 0) { assertEquals(decommissioningNodes.size(), 1); DatanodeDescriptor decommNode = decommissioningNodes.get(0); @@ -222,7 +223,7 @@ public class TestDecommissioningStatus { // This will remove the datanodes from decommissioning list and // make them available again. writeConfigFile(localFileSys, excludeFile, null); - fsn.refreshNodes(conf); + dm.refreshNodes(conf); st1.close(); cleanupFile(fileSys, file1); cleanupFile(fileSys, file2); diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java index 004922f3e9f..9e559463495 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java @@ -22,9 +22,15 @@ import java.io.*; import java.net.URI; import java.util.Collection; import java.util.Iterator; +import java.util.List; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Arrays; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -49,6 +55,10 @@ import org.apache.log4j.Level; import org.aspectj.util.FileUtil; import org.mockito.Mockito; +import org.junit.Test; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import static org.apache.hadoop.test.MetricsAsserts.*; @@ -676,28 +686,44 @@ public class TestEditLog extends TestCase { private static class EditLogByteInputStream extends EditLogInputStream { private InputStream input; private long len; + private int version; + private FSEditLogOp.Reader reader = null; + private FSEditLogLoader.PositionTrackingInputStream tracker = null; - public EditLogByteInputStream(byte[] data) { + public EditLogByteInputStream(byte[] data) throws IOException { len = data.length; input = new ByteArrayInputStream(data); - } - public int available() throws IOException { - return input.available(); - } - - public int read() throws IOException { - return input.read(); + BufferedInputStream bin = new BufferedInputStream(input); + DataInputStream in = new DataInputStream(bin); + version = EditLogFileInputStream.readLogVersion(in); + tracker = new FSEditLogLoader.PositionTrackingInputStream(in); + in = new DataInputStream(tracker); + + reader = new FSEditLogOp.Reader(in, version); } + @Override public long length() throws IOException { return len; } - - public int read(byte[] b, int off, int len) throws IOException { - return input.read(b, off, len); + + @Override + public long getPosition() { + return tracker.getPos(); } + @Override + public FSEditLogOp readOp() throws IOException { + return reader.readOp(); + } + + @Override + public int getVersion() throws IOException { + return version; + } + + @Override public void close() throws IOException { input.close(); } @@ -729,4 +755,103 @@ public class TestEditLog extends TestCase { log.close(); } } + + /** + * Tests the getEditLogManifest function using mock storage for a number + * of different situations. + */ + @Test + public void testEditLogManifestMocks() throws IOException { + NNStorage storage; + FSEditLog log; + // Simple case - different directories have the same + // set of logs, with an in-progress one at end + storage = mockStorageWithEdits( + "[1,100]|[101,200]|[201,]", + "[1,100]|[101,200]|[201,]"); + log = new FSEditLog(storage); + assertEquals("[[1,100], [101,200]]", + log.getEditLogManifest(1).toString()); + assertEquals("[[101,200]]", + log.getEditLogManifest(101).toString()); + + // Another simple case, different directories have different + // sets of files + storage = mockStorageWithEdits( + "[1,100]|[101,200]", + "[1,100]|[201,300]|[301,400]"); // nothing starting at 101 + log = new FSEditLog(storage); + assertEquals("[[1,100], [101,200], [201,300], [301,400]]", + log.getEditLogManifest(1).toString()); + + // Case where one directory has an earlier finalized log, followed + // by a gap. The returned manifest should start after the gap. + storage = mockStorageWithEdits( + "[1,100]|[301,400]", // gap from 101 to 300 + "[301,400]|[401,500]"); + log = new FSEditLog(storage); + assertEquals("[[301,400], [401,500]]", + log.getEditLogManifest(1).toString()); + + // Case where different directories have different length logs + // starting at the same txid - should pick the longer one + storage = mockStorageWithEdits( + "[1,100]|[101,150]", // short log at 101 + "[1,50]|[101,200]"); // short log at 1 + log = new FSEditLog(storage); + assertEquals("[[1,100], [101,200]]", + log.getEditLogManifest(1).toString()); + assertEquals("[[101,200]]", + log.getEditLogManifest(101).toString()); + + // Case where the first storage has an inprogress while + // the second has finalised that file (i.e. the first failed + // recently) + storage = mockStorageWithEdits( + "[1,100]|[101,]", + "[1,100]|[101,200]"); + log = new FSEditLog(storage); + assertEquals("[[1,100], [101,200]]", + log.getEditLogManifest(1).toString()); + assertEquals("[[101,200]]", + log.getEditLogManifest(101).toString()); + } + + /** + * Create a mock NNStorage object with several directories, each directory + * holding edit logs according to a specification. Each directory + * is specified by a pipe-separated string. For example: + * [1,100]|[101,200] specifies a directory which + * includes two finalized segments, one from 1-100, and one from 101-200. + * The syntax [1,] specifies an in-progress log starting at + * txid 1. + */ + private NNStorage mockStorageWithEdits(String... editsDirSpecs) { + List sds = Lists.newArrayList(); + for (String dirSpec : editsDirSpecs) { + List files = Lists.newArrayList(); + String[] logSpecs = dirSpec.split("\\|"); + for (String logSpec : logSpecs) { + Matcher m = Pattern.compile("\\[(\\d+),(\\d+)?\\]").matcher(logSpec); + assertTrue("bad spec: " + logSpec, m.matches()); + if (m.group(2) == null) { + files.add(NNStorage.getInProgressEditsFileName( + Long.valueOf(m.group(1)))); + } else { + files.add(NNStorage.getFinalizedEditsFileName( + Long.valueOf(m.group(1)), + Long.valueOf(m.group(2)))); + } + } + sds.add(FSImageTestUtil.mockStorageDirectory( + NameNodeDirType.EDITS, false, + files.toArray(new String[0]))); + } + + NNStorage storage = Mockito.mock(NNStorage.class); + Mockito.doReturn(sds).when(storage).dirIterable(NameNodeDirType.EDITS); + return storage; + } + + } diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java index b395c542ff1..a673c5f3b37 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java @@ -61,7 +61,7 @@ public class TestEditLogFileOutputStream { .getStorage().getStorageDir(0); File editLog = NNStorage.getInProgressEditsFile(sd, 1); - EditLogValidation validation = FSEditLogLoader.validateEditLog(editLog); + EditLogValidation validation = EditLogFileInputStream.validateEditLog(editLog); assertEquals("Edit log should contain a header as valid length", HEADER_LEN, validation.validLength); assertEquals(1, validation.numTransactions); @@ -73,7 +73,7 @@ public class TestEditLogFileOutputStream { new FsPermission((short)777)); long oldLength = validation.validLength; - validation = FSEditLogLoader.validateEditLog(editLog); + validation = EditLogFileInputStream.validateEditLog(editLog); assertTrue("Edit log should have more valid data after writing a txn " + "(was: " + oldLength + " now: " + validation.validLength + ")", validation.validLength > oldLength); diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java index ab4add30259..a53a0bf26c7 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java @@ -186,7 +186,7 @@ public class TestFSEditLogLoader { // Make sure that uncorrupted log has the expected length and number // of transactions. - EditLogValidation validation = FSEditLogLoader.validateEditLog(logFile); + EditLogValidation validation = EditLogFileInputStream.validateEditLog(logFile); assertEquals(NUM_TXNS + 2, validation.numTransactions); assertEquals(validLength, validation.validLength); @@ -202,7 +202,7 @@ public class TestFSEditLogLoader { // Restore backup, truncate the file exactly before the txn Files.copy(logFileBak, logFile); truncateFile(logFile, txOffset); - validation = FSEditLogLoader.validateEditLog(logFile); + validation = EditLogFileInputStream.validateEditLog(logFile); assertEquals("Failed when truncating to length " + txOffset, txid - 1, validation.numTransactions); assertEquals(txOffset, validation.validLength); @@ -211,7 +211,7 @@ public class TestFSEditLogLoader { // also isn't valid Files.copy(logFileBak, logFile); truncateFile(logFile, txOffset + 1); - validation = FSEditLogLoader.validateEditLog(logFile); + validation = EditLogFileInputStream.validateEditLog(logFile); assertEquals("Failed when truncating to length " + (txOffset + 1), txid - 1, validation.numTransactions); assertEquals(txOffset, validation.validLength); @@ -219,7 +219,7 @@ public class TestFSEditLogLoader { // Restore backup, corrupt the txn opcode Files.copy(logFileBak, logFile); corruptByteInFile(logFile, txOffset); - validation = FSEditLogLoader.validateEditLog(logFile); + validation = EditLogFileInputStream.validateEditLog(logFile); assertEquals("Failed when corrupting txn opcode at " + txOffset, txid - 1, validation.numTransactions); assertEquals(txOffset, validation.validLength); @@ -227,7 +227,7 @@ public class TestFSEditLogLoader { // Restore backup, corrupt a byte a few bytes into the txn Files.copy(logFileBak, logFile); corruptByteInFile(logFile, txOffset+5); - validation = FSEditLogLoader.validateEditLog(logFile); + validation = EditLogFileInputStream.validateEditLog(logFile); assertEquals("Failed when corrupting txn data at " + (txOffset+5), txid - 1, validation.numTransactions); assertEquals(txOffset, validation.validLength); @@ -240,7 +240,7 @@ public class TestFSEditLogLoader { for (long offset = 0; offset < validLength; offset++) { Files.copy(logFileBak, logFile); corruptByteInFile(logFile, offset); - EditLogValidation val = FSEditLogLoader.validateEditLog(logFile); + EditLogValidation val = EditLogFileInputStream.validateEditLog(logFile); assertTrue(val.numTransactions >= prevNumValid); prevNumValid = val.numTransactions; } diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java index 297f737cdeb..113dcbc3393 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java @@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs.server.namenode; import static org.junit.Assert.*; import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import java.io.File; @@ -29,15 +28,14 @@ import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hdfs.server.common.Storage.StorageDirType; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getInProgressEditsFileName; import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getFinalizedEditsFileName; import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getImageFileName; -import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundEditLog; -import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundFSImage; +import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile; +import org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.FSImageFile; import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.TransactionalLoadPlan; import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.LogGroup; import org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.LoadPlan; @@ -56,7 +54,7 @@ public class TestFSImageStorageInspector { FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector(); - StorageDirectory mockDir = mockDirectory( + StorageDirectory mockDir = FSImageTestUtil.mockStorageDirectory( NameNodeDirType.IMAGE_AND_EDITS, false, "/foo/current/" + getImageFileName(123), @@ -72,7 +70,7 @@ public class TestFSImageStorageInspector { assertEquals(2, inspector.foundImages.size()); assertTrue(inspector.foundEditLogs.get(1).isInProgress()); - FoundFSImage latestImage = inspector.getLatestImage(); + FSImageFile latestImage = inspector.getLatestImage(); assertEquals(456, latestImage.txId); assertSame(mockDir, latestImage.sd); assertTrue(inspector.isUpgradeFinalized()); @@ -95,7 +93,7 @@ public class TestFSImageStorageInspector { FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector(); - StorageDirectory mockDir = mockDirectory( + StorageDirectory mockDir = FSImageTestUtil.mockStorageDirectory( NameNodeDirType.IMAGE_AND_EDITS, false, "/foo/current/" + getImageFileName(123), @@ -123,7 +121,7 @@ public class TestFSImageStorageInspector { FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector(); - StorageDirectory mockDir = mockDirectory( + StorageDirectory mockDir = FSImageTestUtil.mockStorageDirectory( NameNodeDirType.IMAGE_AND_EDITS, false, "/foo/current/" + getImageFileName(123), @@ -196,14 +194,14 @@ public class TestFSImageStorageInspector { inspector.inspectDirectory( mockDirectoryWithEditLogs("/foo3/current/" + getInProgressEditsFileName(123))); - inspector.inspectDirectory(mockDirectory( + inspector.inspectDirectory(FSImageTestUtil.mockStorageDirectory( NameNodeDirType.IMAGE, false, "/foo4/current/" + getImageFileName(122))); LogGroup lg = inspector.logGroups.get(123L); assertEquals(3, lg.logs.size()); - FoundEditLog inProgressLog = lg.logs.get(2); + EditLogFile inProgressLog = lg.logs.get(2); assertTrue(inProgressLog.isInProgress()); LoadPlan plan = inspector.createLoadPlan(); @@ -282,7 +280,7 @@ public class TestFSImageStorageInspector { assertTrue(lg.logs.get(2).isCorrupt()); // Calling recover should move it aside - FoundEditLog badLog = lg.logs.get(2); + EditLogFile badLog = lg.logs.get(2); Mockito.doNothing().when(badLog).moveAsideCorruptFile(); Mockito.doNothing().when(lg.logs.get(0)).finalizeLog(); Mockito.doNothing().when(lg.logs.get(1)).finalizeLog(); @@ -303,12 +301,12 @@ public class TestFSImageStorageInspector { String path, int numValidTransactions) throws IOException { for (LogGroup lg : inspector.logGroups.values()) { - List logs = lg.logs; + List logs = lg.logs; for (int i = 0; i < logs.size(); i++) { - FoundEditLog log = logs.get(i); - if (log.file.getPath().equals(path)) { + EditLogFile log = logs.get(i); + if (log.getFile().getPath().equals(path)) { // mock out its validation - FoundEditLog spyLog = spy(log); + EditLogFile spyLog = spy(log); doReturn(new FSEditLogLoader.EditLogValidation(-1, numValidTransactions)) .when(spyLog).validateLog(); logs.set(i, spyLog); @@ -327,15 +325,15 @@ public class TestFSImageStorageInspector { FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector(); - StorageDirectory mockImageDir = mockDirectory( + StorageDirectory mockImageDir = FSImageTestUtil.mockStorageDirectory( NameNodeDirType.IMAGE, false, "/foo/current/" + getImageFileName(123)); - StorageDirectory mockImageDir2 = mockDirectory( + StorageDirectory mockImageDir2 = FSImageTestUtil.mockStorageDirectory( NameNodeDirType.IMAGE, false, "/foo2/current/" + getImageFileName(456)); - StorageDirectory mockEditsDir = mockDirectory( + StorageDirectory mockEditsDir = FSImageTestUtil.mockStorageDirectory( NameNodeDirType.EDITS, false, "/foo3/current/" + getFinalizedEditsFileName(123, 456), @@ -356,7 +354,7 @@ public class TestFSImageStorageInspector { // Check plan TransactionalLoadPlan plan = (TransactionalLoadPlan)inspector.createLoadPlan(); - FoundFSImage pickedImage = plan.image; + FSImageFile pickedImage = plan.image; assertEquals(456, pickedImage.txId); assertSame(mockImageDir2, pickedImage.sd); assertEquals(new File("/foo2/current/" + getImageFileName(456)), @@ -364,43 +362,8 @@ public class TestFSImageStorageInspector { assertArrayEquals(new File[] { new File("/foo3/current/" + getInProgressEditsFileName(457)) }, plan.getEditsFiles().toArray(new File[0])); - - // Check log manifest - assertEquals("[[123,456]]", inspector.getEditLogManifest(123).toString()); - assertEquals("[[123,456]]", inspector.getEditLogManifest(456).toString()); - assertEquals("[]", inspector.getEditLogManifest(457).toString()); } - @Test - public void testLogManifest() throws IOException { - FSImageTransactionalStorageInspector inspector = - new FSImageTransactionalStorageInspector(); - inspector.inspectDirectory( - mockDirectoryWithEditLogs("/foo1/current/" - + getFinalizedEditsFileName(1,1), - "/foo1/current/" - + getFinalizedEditsFileName(2,200))); - inspector.inspectDirectory( - mockDirectoryWithEditLogs("/foo2/current/" - + getInProgressEditsFileName(1), - "/foo2/current/" - + getFinalizedEditsFileName(201, 400))); - inspector.inspectDirectory( - mockDirectoryWithEditLogs("/foo3/current/" - + getFinalizedEditsFileName(1, 1), - "/foo3/current/" - + getFinalizedEditsFileName(2,200))); - - assertEquals("[[1,1], [2,200], [201,400]]", - inspector.getEditLogManifest(1).toString()); - assertEquals("[[2,200], [201,400]]", - inspector.getEditLogManifest(2).toString()); - assertEquals("[[2,200], [201,400]]", - inspector.getEditLogManifest(10).toString()); - assertEquals("[[201,400]]", - inspector.getEditLogManifest(201).toString()); - } - /** * Test case where an in-progress log is in an earlier name directory * than a finalized log. Previously, getEditLogManifest wouldn't @@ -426,46 +389,9 @@ public class TestFSImageStorageInspector { + getFinalizedEditsFileName(2626,2627), "/foo2/current/" + getFinalizedEditsFileName(2628,2629))); - - assertEquals("[[2622,2623], [2624,2625], [2626,2627], [2628,2629]]", - inspector.getEditLogManifest(2621).toString()); } - private StorageDirectory mockDirectoryWithEditLogs(String... fileNames) { - return mockDirectory(NameNodeDirType.EDITS, false, fileNames); - } - - /** - * Make a mock storage directory that returns some set of file contents. - * @param type type of storage dir - * @param previousExists should we mock that the previous/ dir exists? - * @param fileNames the names of files contained in current/ - */ - static StorageDirectory mockDirectory( - StorageDirType type, - boolean previousExists, - String... fileNames) { - StorageDirectory sd = mock(StorageDirectory.class); - - doReturn(type).when(sd).getStorageDirType(); - - // Version file should always exist - doReturn(FSImageTestUtil.mockFile(true)).when(sd).getVersionFile(); - - // Previous dir optionally exists - doReturn(FSImageTestUtil.mockFile(previousExists)) - .when(sd).getPreviousDir(); - - // Return a mock 'current' directory which has the given paths - File[] files = new File[fileNames.length]; - for (int i = 0; i < fileNames.length; i++) { - files[i] = new File(fileNames[i]); - } - - File mockDir = Mockito.spy(new File("/dir/current")); - doReturn(files).when(mockDir).listFiles(); - doReturn(mockDir).when(sd).getCurrentDir(); - - return sd; + static StorageDirectory mockDirectoryWithEditLogs(String... fileNames) { + return FSImageTestUtil.mockStorageDirectory(NameNodeDirType.EDITS, false, fileNames); } } diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java new file mode 100644 index 00000000000..748caf4d4cb --- /dev/null +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import static org.junit.Assert.*; + +import java.io.IOException; + +import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; +import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; +import org.apache.hadoop.test.GenericTestUtils; +import org.junit.Test; + +import com.google.common.base.Joiner; + +public class TestFileJournalManager { + + @Test + public void testGetRemoteEditLog() throws IOException { + StorageDirectory sd = FSImageTestUtil.mockStorageDirectory( + NameNodeDirType.EDITS, false, + NNStorage.getFinalizedEditsFileName(1, 100), + NNStorage.getFinalizedEditsFileName(101, 200), + NNStorage.getInProgressEditsFileName(201), + NNStorage.getFinalizedEditsFileName(1001, 1100)); + + FileJournalManager fjm = new FileJournalManager(sd); + assertEquals("[1,100],[101,200],[1001,1100]", getLogsAsString(fjm, 1)); + assertEquals("[101,200],[1001,1100]", getLogsAsString(fjm, 101)); + assertEquals("[1001,1100]", getLogsAsString(fjm, 201)); + try { + assertEquals("[]", getLogsAsString(fjm, 150)); + fail("Did not throw when asking for a txn in the middle of a log"); + } catch (IOException ioe) { + GenericTestUtils.assertExceptionContains( + "150 which is in the middle", ioe); + } + assertEquals("Asking for a newer log than exists should return empty list", + "", getLogsAsString(fjm, 9999)); + } + + private static String getLogsAsString( + FileJournalManager fjm, long firstTxId) throws IOException { + return Joiner.on(",").join(fjm.getRemoteEditLogs(firstTxId)); + } + +} diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java index aa3ab82b405..aadca5cc20d 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java @@ -24,8 +24,8 @@ import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; -import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundEditLog; -import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundFSImage; +import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile; +import org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.FSImageFile; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getInProgressEditsFileName; import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getFinalizedEditsFileName; @@ -168,14 +168,14 @@ public class TestNNStorageRetentionManager { StoragePurger mockPurger = Mockito.mock(NNStorageRetentionManager.StoragePurger.class); - ArgumentCaptor imagesPurgedCaptor = - ArgumentCaptor.forClass(FoundFSImage.class); - ArgumentCaptor logsPurgedCaptor = - ArgumentCaptor.forClass(FoundEditLog.class); + ArgumentCaptor imagesPurgedCaptor = + ArgumentCaptor.forClass(FSImageFile.class); + ArgumentCaptor logsPurgedCaptor = + ArgumentCaptor.forClass(EditLogFile.class); // Ask the manager to purge files we don't need any more new NNStorageRetentionManager(conf, - tc.mockStorage(), tc.mockEditLog(), mockPurger) + tc.mockStorage(), tc.mockEditLog(mockPurger), mockPurger) .purgeOldStorage(); // Verify that it asked the purger to remove the correct files @@ -186,7 +186,7 @@ public class TestNNStorageRetentionManager { // Check images Set purgedPaths = Sets.newHashSet(); - for (FoundFSImage purged : imagesPurgedCaptor.getAllValues()) { + for (FSImageFile purged : imagesPurgedCaptor.getAllValues()) { purgedPaths.add(purged.getFile().toString()); } Assert.assertEquals(Joiner.on(",").join(tc.expectedPurgedImages), @@ -194,7 +194,7 @@ public class TestNNStorageRetentionManager { // Check images purgedPaths.clear(); - for (FoundEditLog purged : logsPurgedCaptor.getAllValues()) { + for (EditLogFile purged : logsPurgedCaptor.getAllValues()) { purgedPaths.add(purged.getFile().toString()); } Assert.assertEquals(Joiner.on(",").join(tc.expectedPurgedLogs), @@ -216,7 +216,7 @@ public class TestNNStorageRetentionManager { } StorageDirectory mockStorageDir() { - return TestFSImageStorageInspector.mockDirectory( + return FSImageTestUtil.mockStorageDirectory( type, false, files.toArray(new String[0])); } @@ -256,13 +256,14 @@ public class TestNNStorageRetentionManager { return mockStorageForDirs(sds.toArray(new StorageDirectory[0])); } - public FSEditLog mockEditLog() { + public FSEditLog mockEditLog(StoragePurger purger) { final List jms = Lists.newArrayList(); for (FakeRoot root : dirRoots.values()) { if (!root.type.isOfType(NameNodeDirType.EDITS)) continue; FileJournalManager fjm = new FileJournalManager( root.mockStorageDir()); + fjm.purger = purger; jms.add(fjm); } @@ -272,17 +273,15 @@ public class TestNNStorageRetentionManager { @Override public Void answer(InvocationOnMock invocation) throws Throwable { Object[] args = invocation.getArguments(); - assert args.length == 2; + assert args.length == 1; long txId = (Long) args[0]; - StoragePurger purger = (StoragePurger) args[1]; for (JournalManager jm : jms) { - jm.purgeLogsOlderThan(txId, purger); + jm.purgeLogsOlderThan(txId); } return null; } - }).when(mockLog).purgeLogsOlderThan( - Mockito.anyLong(), (StoragePurger) Mockito.anyObject()); + }).when(mockLog).purgeLogsOlderThan(Mockito.anyLong()); return mockLog; } } diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java index a3759888e32..239ff781232 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode; import java.io.File; import java.util.ArrayList; +import java.util.List; import junit.framework.TestCase; @@ -32,6 +33,7 @@ import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; @@ -59,11 +61,13 @@ public class TestNamenodeCapacityReport extends TestCase { cluster.waitActive(); final FSNamesystem namesystem = cluster.getNamesystem(); + final DatanodeManager dm = cluster.getNamesystem().getBlockManager( + ).getDatanodeManager(); // Ensure the data reported for each data node is right - ArrayList live = new ArrayList(); - ArrayList dead = new ArrayList(); - namesystem.DFSNodesStatus(live, dead); + final List live = new ArrayList(); + final List dead = new ArrayList(); + dm.fetchDatanodes(live, dead, false); assertTrue(live.size() == 1); @@ -112,10 +116,10 @@ public class TestNamenodeCapacityReport extends TestCase { configCapacity = namesystem.getCapacityTotal(); used = namesystem.getCapacityUsed(); - nonDFSUsed = namesystem.getCapacityUsedNonDFS(); + nonDFSUsed = namesystem.getNonDfsUsedSpace(); remaining = namesystem.getCapacityRemaining(); - percentUsed = namesystem.getCapacityUsedPercent(); - percentRemaining = namesystem.getCapacityRemainingPercent(); + percentUsed = namesystem.getPercentUsed(); + percentRemaining = namesystem.getPercentRemaining(); bpUsed = namesystem.getBlockPoolUsedSpace(); percentBpUsed = namesystem.getPercentBlockPoolUsed(); diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java index 1b51a18d448..095c153fa23 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java @@ -140,17 +140,9 @@ public class TestStorageRestore { /** * test */ - public void printStorages(FSImage fs) { - LOG.info("current storages and corresponding sizes:"); - for(Iterator it = fs.getStorage().dirIterator(); it.hasNext(); ) { - StorageDirectory sd = it.next(); - - File curDir = sd.getCurrentDir(); - for (File f : curDir.listFiles()) { - LOG.info(" file " + f.getAbsolutePath() + "; len = " + f.length()); - } - } - } + private void printStorages(FSImage image) { + FSImageTestUtil.logStorageContents(LOG, image.getStorage()); + } /** * test diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStreamFile.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStreamFile.java index 360faa33972..989e0b13571 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStreamFile.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStreamFile.java @@ -48,8 +48,8 @@ import org.mockito.Mockito; import org.mortbay.jetty.InclusiveByteRange; /* - Mock input stream class that always outputs the current position of the stream -*/ + * Mock input stream class that always outputs the current position of the stream. + */ class MockFSInputStream extends FSInputStream { long currentPos = 0; public int read() throws IOException { @@ -198,7 +198,7 @@ public class TestStreamFile { } - // Test for positive scenario + // Test for positive scenario @Test public void testDoGetShouldWriteTheFileContentIntoServletOutputStream() throws Exception { @@ -264,9 +264,11 @@ public class TestStreamFile { Mockito.doReturn(CONF).when(mockServletContext).getAttribute( JspHelper.CURRENT_CONF); Mockito.doReturn(NameNode.getHostPortString(NameNode.getAddress(CONF))) - .when(mockHttpServletRequest).getParameter("nnaddr"); + .when(mockHttpServletRequest).getParameter("nnaddr"); Mockito.doReturn(testFile.toString()).when(mockHttpServletRequest) - .getPathInfo(); + .getPathInfo(); + Mockito.doReturn("/streamFile"+testFile.toString()).when(mockHttpServletRequest) + .getRequestURI(); } static Path writeFile(FileSystem fs, Path f) throws IOException { diff --git a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java index 0ad5a19c757..3426a5ad1e3 100644 --- a/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java @@ -75,6 +75,7 @@ public class TestNameNodeMetrics extends TestCase { private DistributedFileSystem fs; private Random rand = new Random(); private FSNamesystem namesystem; + private BlockManager bm; private static Path getTestPath(String fileName) { return new Path(TEST_ROOT_DIR_PATH, fileName); @@ -85,6 +86,7 @@ public class TestNameNodeMetrics extends TestCase { cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(DATANODE_COUNT).build(); cluster.waitActive(); namesystem = cluster.getNamesystem(); + bm = namesystem.getBlockManager(); fs = (DistributedFileSystem) cluster.getFileSystem(); } @@ -167,7 +169,7 @@ public class TestNameNodeMetrics extends TestCase { // Corrupt first replica of the block LocatedBlock block = NameNodeAdapter.getBlockLocations( cluster.getNameNode(), file.toString(), 0, 1).get(0); - namesystem.markBlockAsCorrupt(block.getBlock(), block.getLocations()[0]); + bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[0]); updateMetrics(); MetricsRecordBuilder rb = getMetrics(NS_METRICS); assertGauge("CorruptBlocks", 1L, rb); @@ -188,7 +190,7 @@ public class TestNameNodeMetrics extends TestCase { Path file = getTestPath("testExcessBlocks"); createFile(file, 100, (short)2); long totalBlocks = 1; - namesystem.setReplication(file.toString(), (short)1); + NameNodeAdapter.setReplication(namesystem, file.toString(), (short)1); updateMetrics(); MetricsRecordBuilder rb = getMetrics(NS_METRICS); assertGauge("ExcessBlocks", totalBlocks, rb); @@ -204,7 +206,7 @@ public class TestNameNodeMetrics extends TestCase { // Corrupt the only replica of the block to result in a missing block LocatedBlock block = NameNodeAdapter.getBlockLocations( cluster.getNameNode(), file.toString(), 0, 1).get(0); - namesystem.markBlockAsCorrupt(block.getBlock(), block.getLocations()[0]); + bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[0]); updateMetrics(); MetricsRecordBuilder rb = getMetrics(NS_METRICS); assertGauge("UnderReplicatedBlocks", 1L, rb); diff --git a/hadoop-common/src/test/java/org/apache/hadoop/util/TestCyclicIteration.java b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/util/TestCyclicIteration.java similarity index 98% rename from hadoop-common/src/test/java/org/apache/hadoop/util/TestCyclicIteration.java rename to hdfs/src/test/hdfs/org/apache/hadoop/hdfs/util/TestCyclicIteration.java index 7dfa4763e19..60451eb1ff4 100644 --- a/hadoop-common/src/test/java/org/apache/hadoop/util/TestCyclicIteration.java +++ b/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/util/TestCyclicIteration.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.util; +package org.apache.hadoop.hdfs.util; import java.util.ArrayList; import java.util.Arrays; diff --git a/hdfs/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java b/hdfs/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java index 508a0d2d979..11f695d8f4e 100644 --- a/hdfs/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java +++ b/hdfs/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java @@ -455,7 +455,7 @@ public class TestNNLeaseRecovery { fsn.leaseManager.addLease("mock-lease", file.toString()); if (setStoredBlock) { when(b1.getINode()).thenReturn(iNFmock); - fsn.getBlockManager().blocksMap.addINode(b1, iNFmock); + fsn.getBlockManager().addINode(b1, iNFmock); } when(fsDir.getFileINode(anyString())).thenReturn(iNFmock); diff --git a/mapreduce/CHANGES.txt b/mapreduce/CHANGES.txt index 30fe349403c..1ea594bbcf0 100644 --- a/mapreduce/CHANGES.txt +++ b/mapreduce/CHANGES.txt @@ -38,6 +38,9 @@ Trunk (unreleased changes) MAPREDUCE-2323. Add metrics to the fair scheduler. (todd) + MAPREDUCE-2037. Capture intermediate progress, CPU and memory usage for + tasks. (Dick King via acmurthy) + IMPROVEMENTS MAPREDUCE-2187. Reporter sends progress during sort/merge. (Anupam Seth via @@ -212,6 +215,9 @@ Trunk (unreleased changes) MAPREDUCE-2705. Permits parallel multiple task launches. (Thomas Graves via ddas) + MAPREDUCE-2489. Jobsplits with random hostnames can make the queue + unusable (jeffrey naisbit via mahadev) + OPTIMIZATIONS MAPREDUCE-2026. Make JobTracker.getJobCounters() and @@ -221,6 +227,8 @@ Trunk (unreleased changes) MAPREDUCE-2740. MultipleOutputs in new API creates needless TaskAttemptContexts. (todd) + MAPREDUCE-901. Efficient framework counters. (llu via acmurthy) + BUG FIXES MAPREDUCE-2603. Disable High-Ram emulation in system tests. @@ -381,6 +389,23 @@ Trunk (unreleased changes) MAPREDUCE-2760. mapreduce.jobtracker.split.metainfo.maxsize typoed in mapred-default.xml. (todd via eli) + MAPREDUCE-2797. Update mapreduce tests and RAID for HDFS-2239. (szetszwo) + + MAPREDUCE-2805. Update RAID for HDFS-2241. (szetszwo) + + MAPREDUCE-2837. Ported bug fixes from y-merge to prepare for MAPREDUCE-279 + merge. (acmurthy) + + MAPREDUCE-2541. Fixed a race condition in IndexCache.removeMap. (Binglin + Chang via acmurthy) + + MAPREDUCE-2839. Fixed TokenCache to get delegation tokens using both new + and old apis. (Siddharth Seth via acmurthy) + + MAPREDUCE-2727. Fix divide-by-zero error in SleepJob for sleepCount equals + 0. (Jeffrey Naisbitt via acmurthy) + + Release 0.22.0 - Unreleased INCOMPATIBLE CHANGES diff --git a/mapreduce/src/contrib/raid/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRaid.java b/mapreduce/src/contrib/raid/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRaid.java index d63b0ae92bc..aea009e3e70 100644 --- a/mapreduce/src/contrib/raid/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRaid.java +++ b/mapreduce/src/contrib/raid/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRaid.java @@ -543,7 +543,7 @@ public class BlockPlacementPolicyRaid extends BlockPlacementPolicy { } // remove the prefix String src = parity.substring(prefix.length()); - if (NameNodeRaidUtil.getFileInfo(namesystem.dir, src, true) == null) { + if (NameNodeRaidUtil.getFileInfo(namesystem, src, true) == null) { return null; } return src; @@ -575,7 +575,7 @@ public class BlockPlacementPolicyRaid extends BlockPlacementPolicy { private String getParityFile(String parityPrefix, String src) throws IOException { String parity = parityPrefix + src; - if (NameNodeRaidUtil.getFileInfo(namesystem.dir, parity, true) == null) { + if (NameNodeRaidUtil.getFileInfo(namesystem, parity, true) == null) { return null; } return parity; diff --git a/mapreduce/src/contrib/raid/src/java/org/apache/hadoop/hdfs/server/datanode/RaidBlockSender.java b/mapreduce/src/contrib/raid/src/java/org/apache/hadoop/hdfs/server/datanode/RaidBlockSender.java index f55e69e5fce..337deaeedfa 100644 --- a/mapreduce/src/contrib/raid/src/java/org/apache/hadoop/hdfs/server/datanode/RaidBlockSender.java +++ b/mapreduce/src/contrib/raid/src/java/org/apache/hadoop/hdfs/server/datanode/RaidBlockSender.java @@ -41,7 +41,7 @@ import org.apache.hadoop.util.StringUtils; /** * Reads a block from the disk and sends it to a recipient. */ -public class RaidBlockSender implements java.io.Closeable, FSConstants { +public class RaidBlockSender implements java.io.Closeable { public static final Log LOG = DataNode.LOG; static final Log ClientTraceLog = DataNode.ClientTraceLog; @@ -389,7 +389,7 @@ public class RaidBlockSender implements java.io.Closeable, FSConstants { streamForSendChunks = baseStream; // assure a mininum buffer size. - maxChunksPerPacket = (Math.max(BUFFER_SIZE, + maxChunksPerPacket = (Math.max(FSConstants.IO_FILE_BUFFER_SIZE, MIN_BUFFER_WITH_TRANSFERTO) + bytesPerChecksum - 1)/bytesPerChecksum; @@ -397,7 +397,7 @@ public class RaidBlockSender implements java.io.Closeable, FSConstants { pktSize += checksumSize * maxChunksPerPacket; } else { maxChunksPerPacket = Math.max(1, - (BUFFER_SIZE + bytesPerChecksum - 1)/bytesPerChecksum); + (FSConstants.IO_FILE_BUFFER_SIZE + bytesPerChecksum - 1)/bytesPerChecksum); pktSize += (bytesPerChecksum + checksumSize) * maxChunksPerPacket; } diff --git a/mapreduce/src/examples/org/apache/hadoop/examples/terasort/TeraInputFormat.java b/mapreduce/src/examples/org/apache/hadoop/examples/terasort/TeraInputFormat.java index 1c58ddc43a3..a381aba913f 100644 --- a/mapreduce/src/examples/org/apache/hadoop/examples/terasort/TeraInputFormat.java +++ b/mapreduce/src/examples/org/apache/hadoop/examples/terasort/TeraInputFormat.java @@ -61,19 +61,32 @@ public class TeraInputFormat extends FileInputFormat { private static List lastResult = null; static class TeraFileSplit extends FileSplit { + static private String[] ZERO_LOCATIONS = new String[0]; + private String[] locations; - public TeraFileSplit() {} + + public TeraFileSplit() { + locations = ZERO_LOCATIONS; + } public TeraFileSplit(Path file, long start, long length, String[] hosts) { super(file, start, length, hosts); - locations = hosts; + try { + locations = super.getLocations(); + } catch (IOException e) { + locations = ZERO_LOCATIONS; + } } + + // XXXXXX should this also be null-protected? protected void setLocations(String[] hosts) { locations = hosts; } + @Override public String[] getLocations() { return locations; } + public String toString() { StringBuffer result = new StringBuffer(); result.append(getPath()); diff --git a/mapreduce/src/java/mapred-default.xml b/mapreduce/src/java/mapred-default.xml index e70a6c65715..f0bf665bee7 100644 --- a/mapreduce/src/java/mapred-default.xml +++ b/mapreduce/src/java/mapred-default.xml @@ -32,6 +32,29 @@     + + mapreduce.jobtracker.jobhistory.task.numberprogresssplits + 12 + Every task attempt progresses from 0.0 to 1.0 [unless + it fails or is killed]. We record, for each task attempt, certain + statistics over each twelfth of the progress range. You can change + the number of intervals we divide the entire range of progress into + by setting this property. Higher values give more precision to the + recorded data, but costs more memory in the job tracker at runtime. + Each increment in this attribute costs 16 bytes per running task. + + + + + mapreduce.job.userhistorylocation + + User can specify a location to store the history files of + a particular job. If nothing is specified, the logs are stored in + output directory. The files are stored in "_logs/history/" in the directory. + User can stop logging by giving the value "none". + + + mapreduce.jobtracker.jobhistory.completed.location diff --git a/mapreduce/src/java/org/apache/hadoop/mapred/ACLsManager.java b/mapreduce/src/java/org/apache/hadoop/mapred/ACLsManager.java index 32cdc903ca9..a186e982b03 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapred/ACLsManager.java +++ b/mapreduce/src/java/org/apache/hadoop/mapred/ACLsManager.java @@ -36,7 +36,7 @@ import org.apache.hadoop.security.authorize.AccessControlList; * QueueManager for queue operations. */ @InterfaceAudience.Private -class ACLsManager { +public class ACLsManager { static Log LOG = LogFactory.getLog(ACLsManager.class); // MROwner(user who started this mapreduce cluster)'s ugi @@ -49,7 +49,7 @@ class ACLsManager { private final boolean aclsEnabled; - ACLsManager(Configuration conf, JobACLsManager jobACLsManager, + public ACLsManager(Configuration conf, JobACLsManager jobACLsManager, QueueManager queueManager) throws IOException { mrOwner = UserGroupInformation.getCurrentUser(); @@ -68,7 +68,7 @@ class ACLsManager { this.queueManager = queueManager; } - UserGroupInformation getMROwner() { + public UserGroupInformation getMROwner() { return mrOwner; } @@ -76,7 +76,7 @@ class ACLsManager { return adminAcl; } - JobACLsManager getJobACLsManager() { + public JobACLsManager getJobACLsManager() { return jobACLsManager; } @@ -85,7 +85,7 @@ class ACLsManager { * i.e. either cluster owner or cluster administrator * @return true, if user is an admin */ - boolean isMRAdmin(UserGroupInformation callerUGI) { + public boolean isMRAdmin(UserGroupInformation callerUGI) { if (adminAcl.isUserAllowed(callerUGI)) { return true; } @@ -111,7 +111,7 @@ class ACLsManager { * @param operation the operation for which authorization is needed * @throws AccessControlException */ - void checkAccess(JobInProgress job, UserGroupInformation callerUGI, + public void checkAccess(JobInProgress job, UserGroupInformation callerUGI, Operation operation) throws AccessControlException { String queue = job.getProfile().getQueueName(); diff --git a/mapreduce/src/java/org/apache/hadoop/mapred/Counters.java b/mapreduce/src/java/org/apache/hadoop/mapred/Counters.java index 7e1ed31ef86..30183934e68 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapred/Counters.java +++ b/mapreduce/src/java/org/apache/hadoop/mapred/Counters.java @@ -18,20 +18,9 @@ package org.apache.hadoop.mapred; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; import java.text.ParseException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.IdentityHashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.MissingResourceException; -import java.util.ResourceBundle; -import org.apache.commons.logging.*; +import org.apache.commons.logging.Log; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.io.IntWritable; @@ -40,421 +29,302 @@ import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.mapreduce.FileSystemCounter; +import org.apache.hadoop.mapreduce.counters.AbstractCounterGroup; +import org.apache.hadoop.mapreduce.counters.AbstractCounters; +import org.apache.hadoop.mapreduce.counters.CounterGroupBase; +import org.apache.hadoop.mapreduce.counters.CounterGroupFactory; +import org.apache.hadoop.mapreduce.counters.FrameworkCounterGroup; +import org.apache.hadoop.mapreduce.counters.FileSystemCounterGroup; +import org.apache.hadoop.mapreduce.counters.GenericCounter; +import org.apache.hadoop.mapreduce.counters.Limits; +import static org.apache.hadoop.mapreduce.util.CountersStrings.*; /** * A set of named counters. - * - *

    Counters represent global counters, defined either by the + * + *

    Counters represent global counters, defined either by the * Map-Reduce framework or applications. Each Counter can be of * any {@link Enum} type.

    - * + * *

    Counters are bunched into {@link Group}s, each comprising of - * counters from a particular Enum class. + * counters from a particular Enum class. * @deprecated Use {@link org.apache.hadoop.mapreduce.Counters} instead. */ @Deprecated @InterfaceAudience.Public @InterfaceStability.Stable -public class Counters implements Writable, Iterable { - private static final Log LOG = LogFactory.getLog(Counters.class); - private static final char GROUP_OPEN = '{'; - private static final char GROUP_CLOSE = '}'; - private static final char COUNTER_OPEN = '['; - private static final char COUNTER_CLOSE = ']'; - private static final char UNIT_OPEN = '('; - private static final char UNIT_CLOSE = ')'; - private static char[] charsToEscape = {GROUP_OPEN, GROUP_CLOSE, - COUNTER_OPEN, COUNTER_CLOSE, - UNIT_OPEN, UNIT_CLOSE}; - - //private static Log log = LogFactory.getLog("Counters.class"); - +public class Counters + extends AbstractCounters { + + public Counters() { + super(groupFactory); + } + + public Counters(org.apache.hadoop.mapreduce.Counters newCounters) { + super(newCounters, groupFactory); + } + /** * Downgrade new {@link org.apache.hadoop.mapreduce.Counters} to old Counters * @param newCounters new Counters * @return old Counters instance corresponding to newCounters */ static Counters downgrade(org.apache.hadoop.mapreduce.Counters newCounters) { - Counters oldCounters = new Counters(); - for (org.apache.hadoop.mapreduce.CounterGroup newGroup: newCounters) { - String groupName = newGroup.getName(); - Group oldGroup = oldCounters.getGroup(groupName); - for (org.apache.hadoop.mapreduce.Counter newCounter: newGroup) { - Counter oldCounter = oldGroup.getCounterForName(newCounter.getName()); - oldCounter.setDisplayName(newCounter.getDisplayName()); - oldCounter.increment(newCounter.getValue()); - } - } - return oldCounters; + return new Counters(newCounters); } /** - * A counter record, comprising its name and value. + * A counter record, comprising its name and value. */ - public static class Counter extends org.apache.hadoop.mapreduce.Counter { - - Counter() { - } + public interface Counter extends org.apache.hadoop.mapreduce.Counter { - Counter(String name, String displayName, long value) { - super(name, displayName); - increment(value); - } - - public void setDisplayName(String newName) { - super.setDisplayName(newName); - } - /** * Returns the compact stringified version of the counter in the format * [(actual-name)(display-name)(value)] + * @return the stringified result */ - public synchronized String makeEscapedCompactString() { + String makeEscapedCompactString(); - // First up, obtain the strings that need escaping. This will help us - // determine the buffer length apriori. - String escapedName = escape(getName()); - String escapedDispName = escape(getDisplayName()); - long currentValue = this.getValue(); - int length = escapedName.length() + escapedDispName.length() + 4; - - length += 8; // For the following delimiting characters - StringBuilder builder = new StringBuilder(length); - builder.append(COUNTER_OPEN); - - // Add the counter name - builder.append(UNIT_OPEN); - builder.append(escapedName); - builder.append(UNIT_CLOSE); - - // Add the display name - builder.append(UNIT_OPEN); - builder.append(escapedDispName); - builder.append(UNIT_CLOSE); - - // Add the value - builder.append(UNIT_OPEN); - builder.append(currentValue); - builder.append(UNIT_CLOSE); - - builder.append(COUNTER_CLOSE); - - return builder.toString(); - } - - // Checks for (content) equality of two (basic) counters - @Deprecated - synchronized boolean contentEquals(Counter c) { - return this.equals(c); - } - /** - * What is the current value of this counter? - * @return the current value + * Checks for (content) equality of two (basic) counters + * @param counter to compare + * @return true if content equals + * @deprecated */ - public synchronized long getCounter() { + @Deprecated + boolean contentEquals(Counter counter); + + /** + * @return the value of the counter + */ + long getCounter(); + } + + static class OldCounterImpl extends GenericCounter implements Counter { + + OldCounterImpl() { + } + + OldCounterImpl(String name, String displayName, long value) { + super(name, displayName, value); + } + + @Override + public synchronized String makeEscapedCompactString() { + return toEscapedCompactString(this); + } + + @Override @Deprecated + public boolean contentEquals(Counter counter) { + return equals(counter); + } + + @Override + public long getCounter() { return getValue(); } - } - + /** - * Group of counters, comprising of counters from a particular - * counter {@link Enum} class. + * Group of counters, comprising of counters from a particular + * counter {@link Enum} class. * - *

    Grouphandles localization of the class name and the + *

    Grouphandles localization of the class name and the * counter names.

    */ - public static class Group implements Writable, Iterable { - private String groupName; - private String displayName; - private Map subcounters = new HashMap(); - - // Optional ResourceBundle for localization of group and counter names. - private ResourceBundle bundle = null; - - Group(String groupName) { - try { - bundle = getResourceBundle(groupName); - } - catch (MissingResourceException neverMind) { - } - this.groupName = groupName; - this.displayName = localize("CounterGroupName", groupName); - if (LOG.isDebugEnabled()) { - LOG.debug("Creating group " + groupName + " with " + - (bundle == null ? "nothing" : "bundle")); - } - } - - /** - * Returns the specified resource bundle, or throws an exception. - * @throws MissingResourceException if the bundle isn't found - */ - private static ResourceBundle getResourceBundle(String enumClassName) { - String bundleName = enumClassName.replace('$','_'); - return ResourceBundle.getBundle(bundleName); - } - - /** - * Returns raw name of the group. This is the name of the enum class - * for this group of counters. - */ - public String getName() { - return groupName; - } - - /** - * Returns localized name of the group. This is the same as getName() by - * default, but different if an appropriate ResourceBundle is found. - */ - public String getDisplayName() { - return displayName; - } - - /** - * Set the display name - */ - public void setDisplayName(String displayName) { - this.displayName = displayName; - } - - /** - * Returns the compact stringified version of the group in the format - * {(actual-name)(display-name)(value)[][][]} where [] are compact strings for the - * counters within. - */ - public String makeEscapedCompactString() { - String[] subcountersArray = new String[subcounters.size()]; + public static interface Group extends CounterGroupBase { - // First up, obtain the strings that need escaping. This will help us - // determine the buffer length apriori. - String escapedName = escape(getName()); - String escapedDispName = escape(getDisplayName()); - int i = 0; - int length = escapedName.length() + escapedDispName.length(); - for (Counter counter : subcounters.values()) { - String escapedStr = counter.makeEscapedCompactString(); - subcountersArray[i++] = escapedStr; - length += escapedStr.length(); - } - - length += 6; // for all the delimiting characters below - StringBuilder builder = new StringBuilder(length); - builder.append(GROUP_OPEN); // group start - - // Add the group name - builder.append(UNIT_OPEN); - builder.append(escapedName); - builder.append(UNIT_CLOSE); - - // Add the display name - builder.append(UNIT_OPEN); - builder.append(escapedDispName); - builder.append(UNIT_CLOSE); - - // write the value - for(Counter counter: subcounters.values()) { - builder.append(counter.makeEscapedCompactString()); - } - - builder.append(GROUP_CLOSE); // group end - return builder.toString(); - } - - @Override - public int hashCode() { - return subcounters.hashCode(); - } - - /** - * Checks for (content) equality of Groups - */ - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || obj.getClass() != getClass()) { - return false; - } - boolean isEqual = false; - Group g = (Group) obj; - synchronized (this) { - if (size() == g.size()) { - isEqual = true; - for (Map.Entry entry : subcounters.entrySet()) { - String key = entry.getKey(); - Counter c1 = entry.getValue(); - Counter c2 = g.getCounterForName(key); - if (!c1.contentEquals(c2)) { - isEqual = false; - break; - } - } - } - } - return isEqual; - } - /** - * Returns the value of the specified counter, or 0 if the counter does + * @param counterName the name of the counter + * @return the value of the specified counter, or 0 if the counter does * not exist. */ - public synchronized long getCounter(String counterName) { - for(Counter counter: subcounters.values()) { - if (counter != null && counter.getDisplayName().equals(counterName)) { - return counter.getValue(); - } - } - return 0L; - } - + long getCounter(String counterName); + + /** + * @return the compact stringified version of the group in the format + * {(actual-name)(display-name)(value)[][][]} where [] are compact strings + * for the counters within. + */ + String makeEscapedCompactString(); + /** * Get the counter for the given id and create it if it doesn't exist. * @param id the numeric id of the counter within the group * @param name the internal counter name * @return the counter - * @deprecated use {@link #getCounter(String)} instead + * @deprecated use {@link #findCounter(String)} instead */ @Deprecated - public synchronized Counter getCounter(int id, String name) { - return getCounterForName(name); - } - + Counter getCounter(int id, String name); + /** * Get the counter for the given name and create it if it doesn't exist. * @param name the internal counter name * @return the counter */ - public synchronized Counter getCounterForName(String name) { - Counter result = subcounters.get(name); - if (result == null) { - if (LOG.isDebugEnabled()) { - LOG.debug("Adding " + name); - } - result = new Counter(name, localize(name + ".name", name), 0L); - subcounters.put(name, result); + Counter getCounterForName(String name); + } + + // All the group impls need this for legacy group interface + static long getCounterValue(Group group, String counterName) { + Counter counter = group.findCounter(counterName, false); + if (counter != null) return counter.getValue(); + return 0L; + } + + // Mix the generic group implementation into the Group interface + private static class GenericGroup extends AbstractCounterGroup + implements Group { + + GenericGroup(String name, String displayName, Limits limits) { + super(name, displayName, limits); + } + + @Override + public long getCounter(String counterName) { + return getCounterValue(this, counterName); + } + + @Override + public String makeEscapedCompactString() { + return toEscapedCompactString(this); + } + + @Override + public Counter getCounter(int id, String name) { + return findCounter(name); + } + + @Override + public Counter getCounterForName(String name) { + return findCounter(name); + } + + @Override + protected Counter newCounter(String counterName, String displayName, + long value) { + return new OldCounterImpl(counterName, displayName, value); + } + + @Override + protected Counter newCounter() { + return new OldCounterImpl(); + } + } + + // Mix the framework group implementation into the Group interface + private static class FrameworkGroupImpl> + extends FrameworkCounterGroup implements Group { + + // Mix the framework counter implmementation into the Counter interface + class FrameworkCounterImpl extends FrameworkCounter implements Counter { + + FrameworkCounterImpl(T key) { + super(key); } - return result; - } - - /** - * Returns the number of counters in this group. - */ - public synchronized int size() { - return subcounters.size(); - } - - /** - * Looks up key in the ResourceBundle and returns the corresponding value. - * If the bundle or the key doesn't exist, returns the default value. - */ - private String localize(String key, String defaultValue) { - String result = defaultValue; - if (bundle != null) { - try { - result = bundle.getString(key); - } - catch (MissingResourceException mre) { - } + + @Override + public String makeEscapedCompactString() { + return toEscapedCompactString(this); } - return result; - } - - public synchronized void write(DataOutput out) throws IOException { - Text.writeString(out, displayName); - WritableUtils.writeVInt(out, subcounters.size()); - for(Counter counter: subcounters.values()) { - counter.write(out); + + @Override + public boolean contentEquals(Counter counter) { + return equals(counter); } - } - - public synchronized void readFields(DataInput in) throws IOException { - displayName = Text.readString(in); - subcounters.clear(); - int size = WritableUtils.readVInt(in); - for(int i=0; i < size; i++) { - Counter counter = new Counter(); - counter.readFields(in); - subcounters.put(counter.getName(), counter); + + @Override + public long getCounter() { + return getValue(); } } - public synchronized Iterator iterator() { - return new ArrayList(subcounters.values()).iterator(); + FrameworkGroupImpl(Class cls) { + super(cls); + } + + @Override + public long getCounter(String counterName) { + return getCounterValue(this, counterName); + } + + @Override + public String makeEscapedCompactString() { + return toEscapedCompactString(this); + } + + @Override @Deprecated + public Counter getCounter(int id, String name) { + return findCounter(name); + } + + @Override + public Counter getCounterForName(String name) { + return findCounter(name); + } + + @Override + protected Counter newCounter(T key) { + return new FrameworkCounterImpl(key); } } - - // Map from group name (enum class name) to map of int (enum ordinal) to - // counter record (name-value pair). - private Map counters = new HashMap(); - /** - * A cache from enum values to the associated counter. Dramatically speeds up - * typical usage. - */ - private Map cache = new IdentityHashMap(); - - /** - * Returns the names of all counter classes. - * @return Set of counter names. - */ - public synchronized Collection getGroupNames() { - return counters.keySet(); - } + // Mix the file system counter group implementation into the Group interface + private static class FSGroupImpl extends FileSystemCounterGroup + implements Group { - public synchronized Iterator iterator() { - return counters.values().iterator(); - } + private class FSCounterImpl extends FSCounter implements Counter { + + FSCounterImpl(String scheme, FileSystemCounter key) { + super(scheme, key); + } + + @Override + public String makeEscapedCompactString() { + return toEscapedCompactString(this); + } + + @Override @Deprecated + public boolean contentEquals(Counter counter) { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public long getCounter() { + return getValue(); + } - /** - * Returns the named counter group, or an empty group if there is none - * with the specified name. - */ - public synchronized Group getGroup(String groupName) { - // To provide support for deprecated group names - if (groupName.equals("org.apache.hadoop.mapred.Task$Counter")) { - groupName = "org.apache.hadoop.mapreduce.TaskCounter"; - LOG.warn("Group org.apache.hadoop.mapred.Task$Counter is deprecated." + - " Use org.apache.hadoop.mapreduce.TaskCounter instead"); - } else if (groupName.equals( - "org.apache.hadoop.mapred.JobInProgress$Counter")) { - groupName = "org.apache.hadoop.mapreduce.JobCounter"; - LOG.warn("Group org.apache.hadoop.mapred.JobInProgress$Counter " + - "is deprecated. Use " + - "org.apache.hadoop.mapreduce.JobCounter instead"); } - Group result = counters.get(groupName); - if (result == null) { - result = new Group(groupName); - counters.put(groupName, result); + + @Override + protected Counter newCounter(String scheme, FileSystemCounter key) { + return new FSCounterImpl(scheme, key); } - return result; + + @Override + public long getCounter(String counterName) { + return getCounterValue(this, counterName); + } + + @Override + public String makeEscapedCompactString() { + return toEscapedCompactString(this); + } + + @Override @Deprecated + public Counter getCounter(int id, String name) { + return findCounter(name); + } + + @Override + public Counter getCounterForName(String name) { + return findCounter(name); + } + } - /** - * Find the counter for the given enum. The same enum will always return the - * same counter. - * @param key the counter key - * @return the matching counter object - */ - public synchronized Counter findCounter(Enum key) { - Counter counter = cache.get(key); - if (counter == null) { - Group group = getGroup(key.getDeclaringClass().getName()); - counter = group.getCounterForName(key.toString()); - cache.put(key, counter); - } - return counter; - } - - /** - * Find a counter given the group and the name. - * @param group the name of the group - * @param name the internal name of the counter - * @return the counter for that name - */ public synchronized Counter findCounter(String group, String name) { if (name.equals("MAP_INPUT_BYTES")) { LOG.warn("Counter name MAP_INPUT_BYTES is deprecated. " + @@ -465,16 +335,47 @@ public class Counters implements Writable, Iterable { return getGroup(group).getCounterForName(name); } + /** + * Provide factory methods for counter group factory implementation. + * See also the GroupFactory in + * {@link org.apache.hadoop.mapreduce.Counters mapreduce.Counters} + */ + static class GroupFactory extends CounterGroupFactory { + + @Override + protected > + FrameworkGroupFactory newFrameworkGroupFactory(final Class cls) { + return new FrameworkGroupFactory() { + @Override public Group newGroup(String name) { + return new FrameworkGroupImpl(cls); // impl in this package + } + }; + } + + @Override + protected Group newGenericGroup(String name, String displayName, + Limits limits) { + return new GenericGroup(name, displayName, limits); + } + + @Override + protected Group newFileSystemGroup() { + return new FSGroupImpl(); + } + } + + private static final GroupFactory groupFactory = new GroupFactory(); + /** * Find a counter by using strings * @param group the name of the group * @param id the id of the counter within the group (0 to N-1) * @param name the internal name of the counter * @return the counter for that name - * @deprecated + * @deprecated use {@link findCounter(String, String)} instead */ @Deprecated - public synchronized Counter findCounter(String group, int id, String name) { + public Counter findCounter(String group, int id, String name) { return findCounter(group, name); } @@ -484,10 +385,10 @@ public class Counters implements Writable, Iterable { * @param key identifies a counter * @param amount amount by which counter is to be incremented */ - public synchronized void incrCounter(Enum key, long amount) { + public void incrCounter(Enum key, long amount) { findCounter(key).increment(amount); } - + /** * Increments the specified counter by the specified amount, creating it if * it didn't already exist. @@ -495,27 +396,29 @@ public class Counters implements Writable, Iterable { * @param counter the internal name of the counter * @param amount amount by which counter is to be incremented */ - public synchronized void incrCounter(String group, String counter, long amount) { + public void incrCounter(String group, String counter, long amount) { findCounter(group, counter).increment(amount); } - + /** * Returns current value of the specified counter, or 0 if the counter * does not exist. + * @param key the counter enum to lookup + * @return the counter value or 0 if counter not found */ - public synchronized long getCounter(Enum key) { + public synchronized long getCounter(Enum key) { return findCounter(key).getValue(); } - + /** - * Increments multiple counters by their amounts in another Counters + * Increments multiple counters by their amounts in another Counters * instance. * @param other the other Counters instance */ public synchronized void incrAllCounters(Counters other) { for (Group otherGroup: other) { Group group = getGroup(otherGroup.getName()); - group.displayName = otherGroup.displayName; + group.setDisplayName(otherGroup.getDisplayName()); for (Counter otherCounter : otherGroup) { Counter counter = group.getCounterForName(otherCounter.getName()); counter.setDisplayName(otherCounter.getDisplayName()); @@ -524,8 +427,19 @@ public class Counters implements Writable, Iterable { } } + /** + * @return the total number of counters + * @deprecated use {@link #countCounters()} instead + */ + public int size() { + return countCounters(); + } + /** * Convenience method for computing the sum of two sets of counters. + * @param a the first counters + * @param b the second counters + * @return a new summed counters object */ public static Counters sum(Counters a, Counters b) { Counters counters = new Counters(); @@ -533,55 +447,7 @@ public class Counters implements Writable, Iterable { counters.incrAllCounters(b); return counters; } - - /** - * Returns the total number of counters, by summing the number of counters - * in each group. - */ - public synchronized int size() { - int result = 0; - for (Group group : this) { - result += group.size(); - } - return result; - } - - /** - * Write the set of groups. - * The external format is: - * #groups (groupName group)* - * - * i.e. the number of groups followed by 0 or more groups, where each - * group is of the form: - * - * groupDisplayName #counters (false | true counter)* - * - * where each counter is of the form: - * - * name (false | true displayName) value - */ - public synchronized void write(DataOutput out) throws IOException { - out.writeInt(counters.size()); - for (Group group: counters.values()) { - Text.writeString(out, group.getName()); - group.write(out); - } - } - - /** - * Read a set of groups. - */ - public synchronized void readFields(DataInput in) throws IOException { - int numClasses = in.readInt(); - counters.clear(); - while (numClasses-- > 0) { - String groupName = Text.readString(in); - Group group = new Group(groupName); - group.readFields(in); - counters.put(groupName, group); - } - } - + /** * Logs the current counter values. * @param log The log to use. @@ -591,212 +457,31 @@ public class Counters implements Writable, Iterable { for(Group group: this) { log.info(" " + group.getDisplayName()); for (Counter counter: group) { - log.info(" " + counter.getDisplayName() + "=" + + log.info(" " + counter.getDisplayName() + "=" + counter.getCounter()); - } - } - } - - /** - * Return textual representation of the counter values. - */ - public synchronized String toString() { - StringBuilder sb = new StringBuilder("Counters: " + size()); - for (Group group: this) { - sb.append("\n\t" + group.getDisplayName()); - for (Counter counter: group) { - sb.append("\n\t\t" + counter.getDisplayName() + "=" + - counter.getCounter()); } } - return sb.toString(); } /** - * Convert a counters object into a single line that is easy to parse. - * @return the string with "name=value" for each counter and separated by "," - */ - public synchronized String makeCompactString() { - StringBuffer buffer = new StringBuffer(); - boolean first = true; - for(Group group: this){ - for(Counter counter: group) { - if (first) { - first = false; - } else { - buffer.append(','); - } - buffer.append(group.getDisplayName()); - buffer.append('.'); - buffer.append(counter.getDisplayName()); - buffer.append(':'); - buffer.append(counter.getCounter()); - } - } - return buffer.toString(); - } - - /** - * Represent the counter in a textual format that can be converted back to + * Represent the counter in a textual format that can be converted back to * its object form * @return the string in the following format - * {(groupname)(group-displayname)[(countername)(displayname)(value)][][]}{}{} + * {(groupName)(group-displayName)[(counterName)(displayName)(value)][]*}* */ - public synchronized String makeEscapedCompactString() { - String[] groupsArray = new String[counters.size()]; - int i = 0; - int length = 0; - - // First up, obtain the escaped string for each group so that we can - // determine the buffer length apriori. - for (Group group : this) { - String escapedString = group.makeEscapedCompactString(); - groupsArray[i++] = escapedString; - length += escapedString.length(); - } - - // Now construct the buffer - StringBuilder builder = new StringBuilder(length); - for (String group : groupsArray) { - builder.append(group); - } - return builder.toString(); + public String makeEscapedCompactString() { + return toEscapedCompactString(this); } - // Extracts a block (data enclosed within delimeters) ignoring escape - // sequences. Throws ParseException if an incomplete block is found else - // returns null. - private static String getBlock(String str, char open, char close, - IntWritable index) throws ParseException { - StringBuilder split = new StringBuilder(); - int next = StringUtils.findNext(str, open, StringUtils.ESCAPE_CHAR, - index.get(), split); - split.setLength(0); // clear the buffer - if (next >= 0) { - ++next; // move over '(' - - next = StringUtils.findNext(str, close, StringUtils.ESCAPE_CHAR, - next, split); - if (next >= 0) { - ++next; // move over ')' - index.set(next); - return split.toString(); // found a block - } else { - throw new ParseException("Unexpected end of block", next); - } - } - return null; // found nothing - } - /** - * Convert a stringified counter representation into a counter object. Note - * that the counter can be recovered if its stringified using - * {@link #makeEscapedCompactString()}. - * @return a Counter + * Convert a stringified (by {@link #makeEscapedCompactString()} counter + * representation into a counter object. + * @param compactString to parse + * @return a new counters object + * @throws ParseException */ - public static Counters fromEscapedCompactString(String compactString) - throws ParseException { - Counters counters = new Counters(); - IntWritable index = new IntWritable(0); - - // Get the group to work on - String groupString = - getBlock(compactString, GROUP_OPEN, GROUP_CLOSE, index); - - while (groupString != null) { - IntWritable groupIndex = new IntWritable(0); - - // Get the actual name - String groupName = - getBlock(groupString, UNIT_OPEN, UNIT_CLOSE, groupIndex); - groupName = unescape(groupName); - - // Get the display name - String groupDisplayName = - getBlock(groupString, UNIT_OPEN, UNIT_CLOSE, groupIndex); - groupDisplayName = unescape(groupDisplayName); - - // Get the counters - Group group = counters.getGroup(groupName); - group.setDisplayName(groupDisplayName); - - String counterString = - getBlock(groupString, COUNTER_OPEN, COUNTER_CLOSE, groupIndex); - - while (counterString != null) { - IntWritable counterIndex = new IntWritable(0); - - // Get the actual name - String counterName = - getBlock(counterString, UNIT_OPEN, UNIT_CLOSE, counterIndex); - counterName = unescape(counterName); - - // Get the display name - String counterDisplayName = - getBlock(counterString, UNIT_OPEN, UNIT_CLOSE, counterIndex); - counterDisplayName = unescape(counterDisplayName); - - // Get the value - long value = - Long.parseLong(getBlock(counterString, UNIT_OPEN, UNIT_CLOSE, - counterIndex)); - - // Add the counter - Counter counter = group.getCounterForName(counterName); - counter.setDisplayName(counterDisplayName); - counter.increment(value); - - // Get the next counter - counterString = - getBlock(groupString, COUNTER_OPEN, COUNTER_CLOSE, groupIndex); - } - - groupString = getBlock(compactString, GROUP_OPEN, GROUP_CLOSE, index); - } - return counters; - } - - // Escapes all the delimiters for counters i.e {,[,(,),],} - private static String escape(String string) { - return StringUtils.escapeString(string, StringUtils.ESCAPE_CHAR, - charsToEscape); - } - - // Unescapes all the delimiters for counters i.e {,[,(,),],} - private static String unescape(String string) { - return StringUtils.unEscapeString(string, StringUtils.ESCAPE_CHAR, - charsToEscape); - } - - @Override - public synchronized int hashCode() { - return counters.hashCode(); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || obj.getClass() != getClass()) { - return false; - } - boolean isEqual = false; - Counters other = (Counters) obj; - synchronized (this) { - if (size() == other.size()) { - isEqual = true; - for (Map.Entry entry : this.counters.entrySet()) { - String key = entry.getKey(); - Group sourceGroup = entry.getValue(); - Group targetGroup = other.getGroup(key); - if (!sourceGroup.equals(targetGroup)) { - isEqual = false; - break; - } - } - } - } - return isEqual; + public static Counters fromEscapedCompactString(String compactString) + throws ParseException { + return parseEscapedCompactString(compactString, new Counters()); } } diff --git a/mapreduce/src/java/org/apache/hadoop/mapred/CumulativePeriodicStats.java b/mapreduce/src/java/org/apache/hadoop/mapred/CumulativePeriodicStats.java new file mode 100644 index 00000000000..65f07019b78 --- /dev/null +++ b/mapreduce/src/java/org/apache/hadoop/mapred/CumulativePeriodicStats.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapred; + +/** + * + * This class is a concrete PeriodicStatsAccumulator that deals with + * measurements where the raw data are a measurement of an + * accumulation. The result in each bucket is the estimate + * of the progress-weighted change in that quantity over the + * progress range covered by the bucket. + * + *

    An easy-to-understand example of this kind of quantity would be + * a distance traveled. It makes sense to consider that portion of + * the total travel that can be apportioned to each bucket. + * + */ +class CumulativePeriodicStats extends PeriodicStatsAccumulator { + // int's are acceptable here, even though times are normally + // long's, because these are a difference and an int won't + // overflow for 24 days. Tasks can't run for more than about a + // week for other reasons, and most jobs would be written + int previousValue = 0; + + CumulativePeriodicStats(int count) { + super(count); + } + + /** + * + * accumulates a new reading by keeping a running account of the + * value distance from the beginning of the bucket to the end of + * this reading + */ + @Override + protected void extendInternal(double newProgress, int newValue) { + if (state == null) { + return; + } + + state.currentAccumulation += (double)(newValue - previousValue); + previousValue = newValue; + } +} diff --git a/mapreduce/src/java/org/apache/hadoop/mapred/IndexCache.java b/mapreduce/src/java/org/apache/hadoop/mapred/IndexCache.java index 8718c90b2f4..63c148d5f25 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapred/IndexCache.java +++ b/mapreduce/src/java/org/apache/hadoop/mapred/IndexCache.java @@ -130,12 +130,19 @@ class IndexCache { } /** - * This method removes the map from the cache. It should be called when - * a map output on this tracker is discarded. + * This method removes the map from the cache if index information for this + * map is loaded(size>0), index information entry in cache will not be + * removed if it is in the loading phrase(size=0), this prevents corruption + * of totalMemoryUsed. It should be called when a map output on this tracker + * is discarded. * @param mapId The taskID of this map. */ public void removeMap(String mapId) { - IndexInformation info = cache.remove(mapId); + IndexInformation info = cache.get(mapId); + if ((info != null) && (info.getSize() == 0)) { + return; + } + info = cache.remove(mapId); if (info != null) { totalMemoryUsed.addAndGet(-info.getSize()); if (!queue.remove(mapId)) { @@ -146,6 +153,19 @@ class IndexCache { } } + /** + * This method checks if cache and totolMemoryUsed is consistent. + * It is only used for unit test. + * @return True if cache and totolMemoryUsed is consistent + */ + boolean checkTotalMemoryUsed() { + int totalSize = 0; + for (IndexInformation info : cache.values()) { + totalSize += info.getSize(); + } + return totalSize == totalMemoryUsed.get(); + } + /** * Bring memory usage below totalMemoryAllowed. */ diff --git a/mapreduce/src/java/org/apache/hadoop/mapred/InterTrackerProtocol.java b/mapreduce/src/java/org/apache/hadoop/mapred/InterTrackerProtocol.java index 9887768e8ac..8e9595f003a 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapred/InterTrackerProtocol.java +++ b/mapreduce/src/java/org/apache/hadoop/mapred/InterTrackerProtocol.java @@ -77,8 +77,10 @@ interface InterTrackerProtocol extends VersionedProtocol { * Version 29: Adding user name to the serialized Task for use by TT. * Version 30: Adding available memory and CPU usage information on TT to * TaskTrackerStatus for MAPREDUCE-1218 + * Version 31: Efficient serialization format for Framework counters + * (MAPREDUCE-901) */ - public static final long versionID = 30L; + public static final long versionID = 31L; public final static int TRACKERS_OK = 0; public final static int UNKNOWN_TASKTRACKER = 1; diff --git a/mapreduce/src/java/org/apache/hadoop/mapred/JobACLsManager.java b/mapreduce/src/java/org/apache/hadoop/mapred/JobACLsManager.java index 9e9ad1ea99e..7a9a5f53cfa 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapred/JobACLsManager.java +++ b/mapreduce/src/java/org/apache/hadoop/mapred/JobACLsManager.java @@ -29,7 +29,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; @InterfaceAudience.Private -class JobACLsManager { +public class JobACLsManager { Configuration conf; @@ -37,7 +37,7 @@ class JobACLsManager { this.conf = conf; } - boolean areACLsEnabled() { + public boolean areACLsEnabled() { return conf.getBoolean(MRConfig.MR_ACLS_ENABLED, false); } @@ -86,7 +86,7 @@ class JobACLsManager { * @param jobACL * @throws AccessControlException */ - boolean checkAccess(UserGroupInformation callerUGI, + public boolean checkAccess(UserGroupInformation callerUGI, JobACL jobOperation, String jobOwner, AccessControlList jobACL) { String user = callerUGI.getShortUserName(); diff --git a/mapreduce/src/java/org/apache/hadoop/mapred/JobInProgress.java b/mapreduce/src/java/org/apache/hadoop/mapred/JobInProgress.java index 429d2b01535..0f4a1352ece 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapred/JobInProgress.java +++ b/mapreduce/src/java/org/apache/hadoop/mapred/JobInProgress.java @@ -20,6 +20,7 @@ package org.apache.hadoop.mapred; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.net.UnknownHostException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Collection; @@ -52,6 +53,7 @@ import org.apache.hadoop.mapreduce.JobCounter; import org.apache.hadoop.mapreduce.JobSubmissionFiles; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.TaskType; +import org.apache.hadoop.mapreduce.counters.LimitExceededException; import org.apache.hadoop.mapreduce.jobhistory.JobFinishedEvent; import org.apache.hadoop.mapreduce.jobhistory.JobHistory; import org.apache.hadoop.mapreduce.jobhistory.JobInfoChangeEvent; @@ -622,7 +624,7 @@ public class JobInProgress { * at {@link JobTracker#initJob(JobInProgress)} for more details. */ public synchronized void initTasks() - throws IOException, KillInterruptedException { + throws IOException, KillInterruptedException, UnknownHostException { if (tasksInited.get() || isComplete()) { return; } @@ -653,6 +655,11 @@ public class JobInProgress { checkTaskLimits(); + // Sanity check the locations so we don't create/initialize unnecessary tasks + for (TaskSplitMetaInfo split : taskSplitMetaInfo) { + NetUtils.verifyHostnames(split.getLocations()); + } + jobtracker.getInstrumentation().addWaitingMaps(getJobID(), numMapTasks); jobtracker.getInstrumentation().addWaitingReduces(getJobID(), numReduceTasks); @@ -1285,8 +1292,12 @@ public class JobInProgress { */ private Counters incrementTaskCounters(Counters counters, TaskInProgress[] tips) { - for (TaskInProgress tip : tips) { - counters.incrAllCounters(tip.getCounters()); + try { + for (TaskInProgress tip : tips) { + counters.incrAllCounters(tip.getCounters()); + } + } catch (LimitExceededException e) { + // too many user counters/groups, leaving existing counters intact. } return counters; } @@ -2667,25 +2678,29 @@ public class JobInProgress { status.getTaskTracker(), ttStatus.getHttpPort()); jobHistory.logEvent(tse, status.getTaskID().getJobID()); - + TaskAttemptID statusAttemptID = status.getTaskID(); if (status.getIsMap()){ MapAttemptFinishedEvent mfe = new MapAttemptFinishedEvent( - status.getTaskID(), taskType, TaskStatus.State.SUCCEEDED.toString(), + statusAttemptID, taskType, TaskStatus.State.SUCCEEDED.toString(), status.getMapFinishTime(), status.getFinishTime(), trackerHostname, status.getStateString(), - new org.apache.hadoop.mapreduce.Counters(status.getCounters())); + new org.apache.hadoop.mapreduce.Counters(status.getCounters()), + tip.getSplits(statusAttemptID).burst() + ); jobHistory.logEvent(mfe, status.getTaskID().getJobID()); }else{ ReduceAttemptFinishedEvent rfe = new ReduceAttemptFinishedEvent( - status.getTaskID(), taskType, TaskStatus.State.SUCCEEDED.toString(), + statusAttemptID, taskType, TaskStatus.State.SUCCEEDED.toString(), status.getShuffleFinishTime(), status.getSortFinishTime(), status.getFinishTime(), trackerHostname, status.getStateString(), - new org.apache.hadoop.mapreduce.Counters(status.getCounters())); + new org.apache.hadoop.mapreduce.Counters(status.getCounters()), + tip.getSplits(statusAttemptID).burst() + ); jobHistory.logEvent(rfe, status.getTaskID().getJobID()); @@ -2738,6 +2753,9 @@ public class JobInProgress { retireMap(tip); if ((finishedMapTasks + failedMapTIPs) == (numMapTasks)) { this.status.setMapProgress(1.0f); + if (canLaunchJobCleanupTask()) { + checkCountersLimitsOrFail(); + } } } else { runningReduceTasks -= 1; @@ -2750,6 +2768,9 @@ public class JobInProgress { retireReduce(tip); if ((finishedReduceTasks + failedReduceTIPs) == (numReduceTasks)) { this.status.setReduceProgress(1.0f); + if (canLaunchJobCleanupTask()) { + checkCountersLimitsOrFail(); + } } } decrementSpeculativeCount(wasSpeculating, tip); @@ -2759,6 +2780,19 @@ public class JobInProgress { } return true; } + + /* + * add up the counters and fail the job if it exceeds the limits. + * Make sure we do not recalculate the counters after we fail the job. + * Currently this is taken care by terminateJob() since it does not + * calculate the counters. + */ + private void checkCountersLimitsOrFail() { + Counters counters = getCounters(); + if (counters.limits().violation() != null) { + jobtracker.failJob(this); + } + } private void updateTaskTrackerStats(TaskInProgress tip, TaskTrackerStatus ttStatus, Map trackerStats, DataStatistics overallStats) { @@ -3165,12 +3199,16 @@ public class JobInProgress { taskid, taskType, startTime, taskTrackerName, taskTrackerPort); jobHistory.logEvent(tse, taskid.getJobID()); + + ProgressSplitsBlock splits = tip.getSplits(taskStatus.getTaskID()); - TaskAttemptUnsuccessfulCompletionEvent tue = - new TaskAttemptUnsuccessfulCompletionEvent(taskid, - taskType, taskStatus.getRunState().toString(), - finishTime, - taskTrackerHostName, diagInfo); + TaskAttemptUnsuccessfulCompletionEvent tue = + new TaskAttemptUnsuccessfulCompletionEvent + (taskid, + taskType, taskStatus.getRunState().toString(), + finishTime, + taskTrackerHostName, diagInfo, + splits.burst()); jobHistory.logEvent(tue, taskid.getJobID()); // After this, try to assign tasks with the one after this, so that diff --git a/mapreduce/src/java/org/apache/hadoop/mapred/JobTracker.java b/mapreduce/src/java/org/apache/hadoop/mapred/JobTracker.java index 6e6d9735570..979a030d381 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapred/JobTracker.java +++ b/mapreduce/src/java/org/apache/hadoop/mapred/JobTracker.java @@ -2778,7 +2778,8 @@ public class JobTracker implements MRConstants, InterTrackerProtocol, */ synchronized boolean processHeartbeat( TaskTrackerStatus trackerStatus, - boolean initialContact) { + boolean initialContact) + throws UnknownHostException { getInstrumentation().heartbeat(); diff --git a/mapreduce/src/java/org/apache/hadoop/mapred/LocalJobRunner.java b/mapreduce/src/java/org/apache/hadoop/mapred/LocalJobRunner.java index d28faca70dc..4d05e406177 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapred/LocalJobRunner.java +++ b/mapreduce/src/java/org/apache/hadoop/mapred/LocalJobRunner.java @@ -240,7 +240,7 @@ public class LocalJobRunner implements ClientProtocol { getShortUserName()); TaskRunner.setupChildMapredLocalDirs(map, localConf); - MapOutputFile mapOutput = new MapOutputFile(); + MapOutputFile mapOutput = new MROutputFiles(); mapOutput.setConf(localConf); mapOutputFiles.put(mapId, mapOutput); @@ -404,7 +404,7 @@ public class LocalJobRunner implements ClientProtocol { if (!this.isInterrupted()) { TaskAttemptID mapId = mapIds.get(i); Path mapOut = mapOutputFiles.get(mapId).getOutputFile(); - MapOutputFile localOutputFile = new MapOutputFile(); + MapOutputFile localOutputFile = new MROutputFiles(); localOutputFile.setConf(localConf); Path reduceIn = localOutputFile.getInputFileForWrite(mapId.getTaskID(), diff --git a/mapreduce/src/java/org/apache/hadoop/mapred/MROutputFiles.java b/mapreduce/src/java/org/apache/hadoop/mapred/MROutputFiles.java new file mode 100644 index 00000000000..3f64a10e307 --- /dev/null +++ b/mapreduce/src/java/org/apache/hadoop/mapred/MROutputFiles.java @@ -0,0 +1,226 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapred; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.fs.LocalDirAllocator; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapreduce.MRConfig; + +/** + * Manipulate the working area for the transient store for maps and reduces. + * + * This class is used by map and reduce tasks to identify the directories that + * they need to write to/read from for intermediate files. The callers of + * these methods are from the Child running the Task. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class MROutputFiles extends MapOutputFile { + + private LocalDirAllocator lDirAlloc = + new LocalDirAllocator(MRConfig.LOCAL_DIR); + + public MROutputFiles() { + } + + /** + * Return the path to local map output file created earlier + * + * @return path + * @throws IOException + */ + @Override + public Path getOutputFile() + throws IOException { + return lDirAlloc.getLocalPathToRead(TaskTracker.OUTPUT + Path.SEPARATOR + + MAP_OUTPUT_FILENAME_STRING, getConf()); + } + + /** + * Create a local map output file name. + * + * @param size the size of the file + * @return path + * @throws IOException + */ + @Override + public Path getOutputFileForWrite(long size) + throws IOException { + return lDirAlloc.getLocalPathForWrite(TaskTracker.OUTPUT + Path.SEPARATOR + + MAP_OUTPUT_FILENAME_STRING, size, getConf()); + } + + /** + * Create a local map output file name on the same volume. + */ + @Override + public Path getOutputFileForWriteInVolume(Path existing) { + return new Path(existing.getParent(), MAP_OUTPUT_FILENAME_STRING); + } + + /** + * Return the path to a local map output index file created earlier + * + * @return path + * @throws IOException + */ + @Override + public Path getOutputIndexFile() + throws IOException { + return lDirAlloc.getLocalPathToRead(TaskTracker.OUTPUT + Path.SEPARATOR + + MAP_OUTPUT_FILENAME_STRING + MAP_OUTPUT_INDEX_SUFFIX_STRING, + getConf()); + } + + /** + * Create a local map output index file name. + * + * @param size the size of the file + * @return path + * @throws IOException + */ + @Override + public Path getOutputIndexFileForWrite(long size) + throws IOException { + return lDirAlloc.getLocalPathForWrite(TaskTracker.OUTPUT + Path.SEPARATOR + + MAP_OUTPUT_FILENAME_STRING + MAP_OUTPUT_INDEX_SUFFIX_STRING, + size, getConf()); + } + + /** + * Create a local map output index file name on the same volume. + */ + @Override + public Path getOutputIndexFileForWriteInVolume(Path existing) { + return new Path(existing.getParent(), + MAP_OUTPUT_FILENAME_STRING + MAP_OUTPUT_INDEX_SUFFIX_STRING); + } + + /** + * Return a local map spill file created earlier. + * + * @param spillNumber the number + * @return path + * @throws IOException + */ + @Override + public Path getSpillFile(int spillNumber) + throws IOException { + return lDirAlloc.getLocalPathToRead(TaskTracker.OUTPUT + "/spill" + + spillNumber + ".out", getConf()); + } + + /** + * Create a local map spill file name. + * + * @param spillNumber the number + * @param size the size of the file + * @return path + * @throws IOException + */ + @Override + public Path getSpillFileForWrite(int spillNumber, long size) + throws IOException { + return lDirAlloc.getLocalPathForWrite(TaskTracker.OUTPUT + "/spill" + + spillNumber + ".out", size, getConf()); + } + + /** + * Return a local map spill index file created earlier + * + * @param spillNumber the number + * @return path + * @throws IOException + */ + @Override + public Path getSpillIndexFile(int spillNumber) + throws IOException { + return lDirAlloc.getLocalPathToRead(TaskTracker.OUTPUT + "/spill" + + spillNumber + ".out.index", getConf()); + } + + /** + * Create a local map spill index file name. + * + * @param spillNumber the number + * @param size the size of the file + * @return path + * @throws IOException + */ + @Override + public Path getSpillIndexFileForWrite(int spillNumber, long size) + throws IOException { + return lDirAlloc.getLocalPathForWrite(TaskTracker.OUTPUT + "/spill" + + spillNumber + ".out.index", size, getConf()); + } + + /** + * Return a local reduce input file created earlier + * + * @param mapId a map task id + * @return path + * @throws IOException + */ + @Override + public Path getInputFile(int mapId) + throws IOException { + return lDirAlloc.getLocalPathToRead(String.format( + REDUCE_INPUT_FILE_FORMAT_STRING, TaskTracker.OUTPUT, Integer + .valueOf(mapId)), getConf()); + } + + /** + * Create a local reduce input file name. + * + * @param mapId a map task id + * @param size the size of the file + * @return path + * @throws IOException + */ + @Override + public Path getInputFileForWrite(org.apache.hadoop.mapreduce.TaskID mapId, + long size) + throws IOException { + return lDirAlloc.getLocalPathForWrite(String.format( + REDUCE_INPUT_FILE_FORMAT_STRING, TaskTracker.OUTPUT, mapId.getId()), + size, getConf()); + } + + /** Removes all of the files related to a task. */ + @Override + public void removeAll() + throws IOException { + ((JobConf)getConf()).deleteLocalFiles(TaskTracker.OUTPUT); + } + + @Override + public void setConf(Configuration conf) { + if (!(conf instanceof JobConf)) { + conf = new JobConf(conf); + } + super.setConf(conf); + } + +} diff --git a/mapreduce/src/java/org/apache/hadoop/mapred/MapOutputFile.java b/mapreduce/src/java/org/apache/hadoop/mapred/MapOutputFile.java index 3d4511b40b9..b5ff73eb1bd 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapred/MapOutputFile.java +++ b/mapreduce/src/java/org/apache/hadoop/mapred/MapOutputFile.java @@ -23,9 +23,8 @@ import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.LocalDirAllocator; +import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapreduce.MRConfig; /** * Manipulate the working area for the transient store for maps and reduces. @@ -38,164 +37,132 @@ import org.apache.hadoop.mapreduce.MRConfig; */ @InterfaceAudience.Private @InterfaceStability.Unstable -public class MapOutputFile { +public abstract class MapOutputFile implements Configurable { - private JobConf conf; + private Configuration conf; + static final String MAP_OUTPUT_FILENAME_STRING = "file.out"; + static final String MAP_OUTPUT_INDEX_SUFFIX_STRING = ".index"; static final String REDUCE_INPUT_FILE_FORMAT_STRING = "%s/map_%d.out"; public MapOutputFile() { } - private LocalDirAllocator lDirAlloc = - new LocalDirAllocator(MRConfig.LOCAL_DIR); - /** * Return the path to local map output file created earlier - * + * * @return path * @throws IOException */ - public Path getOutputFile() - throws IOException { - return lDirAlloc.getLocalPathToRead(TaskTracker.OUTPUT + Path.SEPARATOR - + "file.out", conf); - } + public abstract Path getOutputFile() throws IOException; /** * Create a local map output file name. - * + * * @param size the size of the file * @return path * @throws IOException */ - public Path getOutputFileForWrite(long size) - throws IOException { - return lDirAlloc.getLocalPathForWrite(TaskTracker.OUTPUT + Path.SEPARATOR - + "file.out", size, conf); - } + public abstract Path getOutputFileForWrite(long size) throws IOException; + + /** + * Create a local map output file name on the same volume. + */ + public abstract Path getOutputFileForWriteInVolume(Path existing); /** * Return the path to a local map output index file created earlier - * + * * @return path * @throws IOException */ - public Path getOutputIndexFile() - throws IOException { - return lDirAlloc.getLocalPathToRead(TaskTracker.OUTPUT + Path.SEPARATOR - + "file.out.index", conf); - } + public abstract Path getOutputIndexFile() throws IOException; /** * Create a local map output index file name. - * + * * @param size the size of the file * @return path * @throws IOException */ - public Path getOutputIndexFileForWrite(long size) - throws IOException { - return lDirAlloc.getLocalPathForWrite(TaskTracker.OUTPUT + Path.SEPARATOR - + "file.out.index", size, conf); - } + public abstract Path getOutputIndexFileForWrite(long size) throws IOException; + + /** + * Create a local map output index file name on the same volume. + */ + public abstract Path getOutputIndexFileForWriteInVolume(Path existing); /** * Return a local map spill file created earlier. - * + * * @param spillNumber the number * @return path * @throws IOException */ - public Path getSpillFile(int spillNumber) - throws IOException { - return lDirAlloc.getLocalPathToRead(TaskTracker.OUTPUT + "/spill" - + spillNumber + ".out", conf); - } + public abstract Path getSpillFile(int spillNumber) throws IOException; /** * Create a local map spill file name. - * + * * @param spillNumber the number * @param size the size of the file * @return path * @throws IOException */ - public Path getSpillFileForWrite(int spillNumber, long size) - throws IOException { - return lDirAlloc.getLocalPathForWrite(TaskTracker.OUTPUT + "/spill" - + spillNumber + ".out", size, conf); - } + public abstract Path getSpillFileForWrite(int spillNumber, long size) + throws IOException; /** * Return a local map spill index file created earlier - * + * * @param spillNumber the number * @return path * @throws IOException */ - public Path getSpillIndexFile(int spillNumber) - throws IOException { - return lDirAlloc.getLocalPathToRead(TaskTracker.OUTPUT + "/spill" - + spillNumber + ".out.index", conf); - } + public abstract Path getSpillIndexFile(int spillNumber) throws IOException; /** * Create a local map spill index file name. - * + * * @param spillNumber the number * @param size the size of the file * @return path * @throws IOException */ - public Path getSpillIndexFileForWrite(int spillNumber, long size) - throws IOException { - return lDirAlloc.getLocalPathForWrite(TaskTracker.OUTPUT + "/spill" - + spillNumber + ".out.index", size, conf); - } + public abstract Path getSpillIndexFileForWrite(int spillNumber, long size) + throws IOException; /** * Return a local reduce input file created earlier - * + * * @param mapId a map task id * @return path - * @throws IOException + * @throws IOException */ - public Path getInputFile(int mapId) - throws IOException { - return lDirAlloc.getLocalPathToRead(String.format( - REDUCE_INPUT_FILE_FORMAT_STRING, TaskTracker.OUTPUT, Integer - .valueOf(mapId)), conf); - } + public abstract Path getInputFile(int mapId) throws IOException; /** * Create a local reduce input file name. - * + * * @param mapId a map task id * @param size the size of the file * @return path * @throws IOException */ - public Path getInputFileForWrite(org.apache.hadoop.mapreduce.TaskID mapId, - long size) - throws IOException { - return lDirAlloc.getLocalPathForWrite(String.format( - REDUCE_INPUT_FILE_FORMAT_STRING, TaskTracker.OUTPUT, mapId.getId()), - size, conf); - } + public abstract Path getInputFileForWrite( + org.apache.hadoop.mapreduce.TaskID mapId, long size) throws IOException; /** Removes all of the files related to a task. */ - public void removeAll() - throws IOException { - conf.deleteLocalFiles(TaskTracker.OUTPUT); + public abstract void removeAll() throws IOException; + + @Override + public void setConf(Configuration conf) { + this.conf = conf; } - public void setConf(Configuration conf) { - if (conf instanceof JobConf) { - this.conf = (JobConf) conf; - } else { - this.conf = new JobConf(conf); - } + @Override + public Configuration getConf() { + return conf; } - + } diff --git a/mapreduce/src/java/org/apache/hadoop/mapred/MapTask.java b/mapreduce/src/java/org/apache/hadoop/mapred/MapTask.java index 951b45ae70f..d1bff52ef3f 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapred/MapTask.java +++ b/mapreduce/src/java/org/apache/hadoop/mapred/MapTask.java @@ -1735,13 +1735,13 @@ class MapTask extends Task { } if (numSpills == 1) { //the spill is the final output rfs.rename(filename[0], - new Path(filename[0].getParent(), "file.out")); + mapOutputFile.getOutputFileForWriteInVolume(filename[0])); if (indexCacheList.size() == 0) { rfs.rename(mapOutputFile.getSpillIndexFile(0), - new Path(filename[0].getParent(),"file.out.index")); + mapOutputFile.getOutputIndexFileForWriteInVolume(filename[0])); } else { indexCacheList.get(0).writeToFile( - new Path(filename[0].getParent(),"file.out.index"), job); + mapOutputFile.getOutputIndexFileForWriteInVolume(filename[0]), job); } return; } diff --git a/mapreduce/src/java/org/apache/hadoop/mapred/PeriodicStatsAccumulator.java b/mapreduce/src/java/org/apache/hadoop/mapred/PeriodicStatsAccumulator.java new file mode 100644 index 00000000000..85ee8a544e7 --- /dev/null +++ b/mapreduce/src/java/org/apache/hadoop/mapred/PeriodicStatsAccumulator.java @@ -0,0 +1,205 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapred; + +/** + * + * This abstract class that represents a bucketed series of + * measurements of a quantity being measured in a running task + * attempt. + * + *

    The sole constructor is called with a count, which is the + * number of buckets into which we evenly divide the spectrum of + * progress from 0.0D to 1.0D . In the future we may provide for + * custom split points that don't have to be uniform. + * + *

    A subclass determines how we fold readings for portions of a + * bucket and how we interpret the readings by overriding + * {@code extendInternal(...)} and {@code initializeInterval()} + */ +public abstract class PeriodicStatsAccumulator { + // The range of progress from 0.0D through 1.0D is divided into + // count "progress segments". This object accumulates an + // estimate of the effective value of a time-varying value during + // the zero-based i'th progress segment, ranging from i/count + // through (i+1)/count . + // This is an abstract class. We have two implementations: one + // for monotonically increasing time-dependent variables + // [currently, CPU time in milliseconds and wallclock time in + // milliseconds] and one for quantities that can vary arbitrarily + // over time, currently virtual and physical memory used, in + // kilobytes. + // We carry int's here. This saves a lot of JVM heap space in the + // job tracker per running task attempt [200 bytes per] but it + // has a small downside. + // No task attempt can run for more than 57 days nor occupy more + // than two terabytes of virtual memory. + protected final int count; + protected final int[] values; + + static class StatsetState { + int oldValue = 0; + double oldProgress = 0.0D; + + double currentAccumulation = 0.0D; + } + + // We provide this level of indirection to reduce the memory + // footprint of done task attempts. When a task's progress + // reaches 1.0D, we delete this objecte StatsetState. + StatsetState state = new StatsetState(); + + PeriodicStatsAccumulator(int count) { + this.count = count; + this.values = new int[count]; + for (int i = 0; i < count; ++i) { + values[i] = -1; + } + } + + protected int[] getValues() { + return values; + } + + // The concrete implementation of this abstract function + // accumulates more data into the current progress segment. + // newProgress [from the call] and oldProgress [from the object] + // must be in [or at the border of] a single progress segment. + /** + * + * adds a new reading to the current bucket. + * + * @param newProgress the endpoint of the interval this new + * reading covers + * @param newValue the value of the reading at {@code newProgress} + * + * The class has three instance variables, {@code oldProgress} and + * {@code oldValue} and {@code currentAccumulation}. + * + * {@code extendInternal} can count on three things: + * + * 1: The first time it's called in a particular instance, both + * oldXXX's will be zero. + * + * 2: oldXXX for a later call is the value of newXXX of the + * previous call. This ensures continuity in accumulation from + * one call to the next. + * + * 3: {@code currentAccumulation} is owned by + * {@code initializeInterval} and {@code extendInternal}. + */ + protected abstract void extendInternal(double newProgress, int newValue); + + // What has to be done when you open a new interval + /** + * initializes the state variables to be ready for a new interval + */ + protected void initializeInterval() { + state.currentAccumulation = 0.0D; + } + + // called for each new reading + /** + * This method calls {@code extendInternal} at least once. It + * divides the current progress interval [from the last call's + * {@code newProgress} to this call's {@code newProgress} ] + * into one or more subintervals by splitting at any point which + * is an interval boundary if there are any such points. It + * then calls {@code extendInternal} for each subinterval, or the + * whole interval if there are no splitting points. + * + *

    For example, if the value was {@code 300} last time with + * {@code 0.3} progress, and count is {@code 5}, and you get a + * new reading with the variable at {@code 700} and progress at + * {@code 0.7}, you get three calls to {@code extendInternal}: + * one extending from progress {@code 0.3} to {@code 0.4} [the + * next boundary] with a value of {@code 400}, the next one + * through {@code 0.6} with a value of {@code 600}, and finally + * one at {@code 700} with a progress of {@code 0.7} . + * + * @param newProgress the endpoint of the progress range this new + * reading covers + * @param newValue the value of the reading at {@code newProgress} + */ + protected void extend(double newProgress, int newValue) { + if (state == null || newProgress < state.oldProgress) { + return; + } + + // This correctness of this code depends on 100% * count = count. + int oldIndex = (int)(state.oldProgress * count); + int newIndex = (int)(newProgress * count); + int originalOldValue = state.oldValue; + + double fullValueDistance = (double)newValue - state.oldValue; + double fullProgressDistance = newProgress - state.oldProgress; + double originalOldProgress = state.oldProgress; + + // In this loop we detect each subinterval boundary within the + // range from the old progress to the new one. Then we + // interpolate the value from the old value to the new one to + // infer what its value might have been at each such boundary. + // Lastly we make the necessary calls to extendInternal to fold + // in the data for each trapazoid where no such trapazoid + // crosses a boundary. + for (int closee = oldIndex; closee < newIndex; ++closee) { + double interpolationProgress = (double)(closee + 1) / count; + // In floats, x * y / y might not equal y. + interpolationProgress = Math.min(interpolationProgress, newProgress); + + double progressLength = (interpolationProgress - originalOldProgress); + double interpolationProportion = progressLength / fullProgressDistance; + + double interpolationValueDistance + = fullValueDistance * interpolationProportion; + + // estimates the value at the next [interpolated] subsegment boundary + int interpolationValue + = (int)interpolationValueDistance + originalOldValue; + + extendInternal(interpolationProgress, interpolationValue); + + advanceState(interpolationProgress, interpolationValue); + + values[closee] = (int)state.currentAccumulation; + initializeInterval(); + + } + + extendInternal(newProgress, newValue); + advanceState(newProgress, newValue); + + if (newIndex == count) { + state = null; + } + } + + protected void advanceState(double newProgress, int newValue) { + state.oldValue = newValue; + state.oldProgress = newProgress; + } + + int getCount() { + return count; + } + + int get(int index) { + return values[index]; + } +} diff --git a/mapreduce/src/java/org/apache/hadoop/mapred/ProgressSplitsBlock.java b/mapreduce/src/java/org/apache/hadoop/mapred/ProgressSplitsBlock.java new file mode 100644 index 00000000000..d3912438527 --- /dev/null +++ b/mapreduce/src/java/org/apache/hadoop/mapred/ProgressSplitsBlock.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapred; + +import java.util.List; + +/* + * This object gathers the [currently four] PeriodStatset's that we + * are gathering for a particular task attempt for packaging and + * handling as a single object. + */ +public class ProgressSplitsBlock { + final PeriodicStatsAccumulator progressWallclockTime; + final PeriodicStatsAccumulator progressCPUTime; + final PeriodicStatsAccumulator progressVirtualMemoryKbytes; + final PeriodicStatsAccumulator progressPhysicalMemoryKbytes; + + static final int[] NULL_ARRAY = new int[0]; + + static final int WALLCLOCK_TIME_INDEX = 0; + static final int CPU_TIME_INDEX = 1; + static final int VIRTUAL_MEMORY_KBYTES_INDEX = 2; + static final int PHYSICAL_MEMORY_KBYTES_INDEX = 3; + + static final int DEFAULT_NUMBER_PROGRESS_SPLITS = 12; + + ProgressSplitsBlock(int numberSplits) { + progressWallclockTime + = new CumulativePeriodicStats(numberSplits); + progressCPUTime + = new CumulativePeriodicStats(numberSplits); + progressVirtualMemoryKbytes + = new StatePeriodicStats(numberSplits); + progressPhysicalMemoryKbytes + = new StatePeriodicStats(numberSplits); + } + + // this coordinates with LoggedTaskAttempt.SplitVectorKind + int[][] burst() { + int[][] result = new int[4][]; + + result[WALLCLOCK_TIME_INDEX] = progressWallclockTime.getValues(); + result[CPU_TIME_INDEX] = progressCPUTime.getValues(); + result[VIRTUAL_MEMORY_KBYTES_INDEX] = progressVirtualMemoryKbytes.getValues(); + result[PHYSICAL_MEMORY_KBYTES_INDEX] = progressPhysicalMemoryKbytes.getValues(); + + return result; + } + + static public int[] arrayGet(int[][] burstedBlock, int index) { + return burstedBlock == null ? NULL_ARRAY : burstedBlock[index]; + } + + static public int[] arrayGetWallclockTime(int[][] burstedBlock) { + return arrayGet(burstedBlock, WALLCLOCK_TIME_INDEX); + } + + static public int[] arrayGetCPUTime(int[][] burstedBlock) { + return arrayGet(burstedBlock, CPU_TIME_INDEX); + } + + static public int[] arrayGetVMemKbytes(int[][] burstedBlock) { + return arrayGet(burstedBlock, VIRTUAL_MEMORY_KBYTES_INDEX); + } + + static public int[] arrayGetPhysMemKbytes(int[][] burstedBlock) { + return arrayGet(burstedBlock, PHYSICAL_MEMORY_KBYTES_INDEX); + } +} + diff --git a/mapreduce/src/java/org/apache/hadoop/mapred/ReduceTask.java b/mapreduce/src/java/org/apache/hadoop/mapred/ReduceTask.java index 6256c662730..5e6822a0869 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapred/ReduceTask.java +++ b/mapreduce/src/java/org/apache/hadoop/mapred/ReduceTask.java @@ -362,7 +362,8 @@ public class ReduceTask extends Task { shuffledMapsCounter, reduceShuffleBytes, failedShuffleCounter, mergedMapOutputsCounter, - taskStatus, copyPhase, sortPhase, this); + taskStatus, copyPhase, sortPhase, this, + mapOutputFile); rIter = shuffle.run(); } else { // local job runner doesn't have a copy phase diff --git a/mapreduce/src/java/org/apache/hadoop/mapred/StatePeriodicStats.java b/mapreduce/src/java/org/apache/hadoop/mapred/StatePeriodicStats.java new file mode 100644 index 00000000000..e9577b303b2 --- /dev/null +++ b/mapreduce/src/java/org/apache/hadoop/mapred/StatePeriodicStats.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapred; + + +/** + * + * This class is a concrete PeriodicStatsAccumulator that deals with + * measurements where the raw data are a measurement of a + * time-varying quantity. The result in each bucket is the estimate + * of the progress-weighted mean value of that quantity over the + * progress range covered by the bucket. + * + *

    An easy-to-understand example of this kind of quantity would be + * a temperature. It makes sense to consider the mean temperature + * over a progress range. + * + */ +class StatePeriodicStats extends PeriodicStatsAccumulator { + StatePeriodicStats(int count) { + super(count); + } + + /** + * + * accumulates a new reading by keeping a running account of the + * area under the piecewise linear curve marked by pairs of + * {@code newProgress, newValue} . + */ + @Override + protected void extendInternal(double newProgress, int newValue) { + if (state == null) { + return; + } + + // the effective height of this trapezoid if rectangularized + double mean = ((double)newValue + (double)state.oldValue)/2.0D; + + // conceptually mean * (newProgress - state.oldProgress) / (1 / count) + state.currentAccumulation += mean * (newProgress - state.oldProgress) * count; + } +} diff --git a/mapreduce/src/java/org/apache/hadoop/mapred/Task.java b/mapreduce/src/java/org/apache/hadoop/mapred/Task.java index 8ad56a7d051..449ac0790a1 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapred/Task.java +++ b/mapreduce/src/java/org/apache/hadoop/mapred/Task.java @@ -31,7 +31,6 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.concurrent.atomic.AtomicBoolean; - import javax.crypto.SecretKey; import org.apache.commons.logging.Log; @@ -54,6 +53,7 @@ import org.apache.hadoop.io.serializer.Deserializer; import org.apache.hadoop.io.serializer.SerializationFactory; import org.apache.hadoop.mapred.IFile.Writer; import org.apache.hadoop.mapreduce.Counter; +import org.apache.hadoop.mapreduce.FileSystemCounter; import org.apache.hadoop.mapreduce.OutputCommitter; import org.apache.hadoop.mapreduce.TaskCounter; import org.apache.hadoop.mapreduce.JobStatus; @@ -146,7 +146,7 @@ abstract public class Task implements Writable, Configurable { private long initCpuCumulativeTime = 0; protected JobConf conf; - protected MapOutputFile mapOutputFile = new MapOutputFile(); + protected MapOutputFile mapOutputFile; protected LocalDirAllocator lDirAlloc; private final static int MAX_RETRIES = 10; protected JobContext jobContext; @@ -819,37 +819,41 @@ abstract public class Task implements Writable, Configurable { * system and only creates the counters when they are needed. */ class FileSystemStatisticUpdater { - private long prevReadBytes = 0; - private long prevWriteBytes = 0; private FileSystem.Statistics stats; - private Counters.Counter readCounter = null; - private Counters.Counter writeCounter = null; - private String[] counterNames; + private Counters.Counter readBytesCounter, writeBytesCounter, + readOpsCounter, largeReadOpsCounter, writeOpsCounter; - FileSystemStatisticUpdater(String uriScheme, FileSystem.Statistics stats) { + FileSystemStatisticUpdater(FileSystem.Statistics stats) { this.stats = stats; - this.counterNames = getFileSystemCounterNames(uriScheme); } void updateCounters() { - long newReadBytes = stats.getBytesRead(); - long newWriteBytes = stats.getBytesWritten(); - if (prevReadBytes != newReadBytes) { - if (readCounter == null) { - readCounter = counters.findCounter(FILESYSTEM_COUNTER_GROUP, - counterNames[0]); - } - readCounter.increment(newReadBytes - prevReadBytes); - prevReadBytes = newReadBytes; + String scheme = stats.getScheme(); + if (readBytesCounter == null) { + readBytesCounter = counters.findCounter(scheme, + FileSystemCounter.BYTES_READ); } - if (prevWriteBytes != newWriteBytes) { - if (writeCounter == null) { - writeCounter = counters.findCounter(FILESYSTEM_COUNTER_GROUP, - counterNames[1]); - } - writeCounter.increment(newWriteBytes - prevWriteBytes); - prevWriteBytes = newWriteBytes; + readBytesCounter.setValue(stats.getBytesRead()); + if (writeBytesCounter == null) { + writeBytesCounter = counters.findCounter(scheme, + FileSystemCounter.BYTES_WRITTEN); } + writeBytesCounter.setValue(stats.getBytesWritten()); + if (readOpsCounter == null) { + readOpsCounter = counters.findCounter(scheme, + FileSystemCounter.READ_OPS); + } + readOpsCounter.setValue(stats.getReadOps()); + if (largeReadOpsCounter == null) { + largeReadOpsCounter = counters.findCounter(scheme, + FileSystemCounter.LARGE_READ_OPS); + } + largeReadOpsCounter.setValue(stats.getLargeReadOps()); + if (writeOpsCounter == null) { + writeOpsCounter = counters.findCounter(scheme, + FileSystemCounter.WRITE_OPS); + } + writeOpsCounter.setValue(stats.getWriteOps()); } } @@ -864,7 +868,7 @@ abstract public class Task implements Writable, Configurable { String uriScheme = stat.getScheme(); FileSystemStatisticUpdater updater = statisticUpdaters.get(uriScheme); if(updater==null) {//new FileSystem has been found in the cache - updater = new FileSystemStatisticUpdater(uriScheme, stat); + updater = new FileSystemStatisticUpdater(stat); statisticUpdaters.put(uriScheme, updater); } updater.updateCounters(); @@ -1150,7 +1154,9 @@ abstract public class Task implements Writable, Configurable { } else { this.conf = new JobConf(conf); } - this.mapOutputFile.setConf(this.conf); + this.mapOutputFile = ReflectionUtils.newInstance( + conf.getClass(MRConfig.TASK_LOCAL_OUTPUT_CLASS, + MROutputFiles.class, MapOutputFile.class), conf); this.lDirAlloc = new LocalDirAllocator(MRConfig.LOCAL_DIR); // add the static resolutions (this is required for the junit to // work on testcases that simulate multiple nodes on a single physical diff --git a/mapreduce/src/java/org/apache/hadoop/mapred/TaskInProgress.java b/mapreduce/src/java/org/apache/hadoop/mapred/TaskInProgress.java index 0a703c90658..051e3750cd7 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapred/TaskInProgress.java +++ b/mapreduce/src/java/org/apache/hadoop/mapred/TaskInProgress.java @@ -31,25 +31,32 @@ import java.util.TreeSet; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; + import org.apache.hadoop.mapred.JobInProgress.DataStatistics; import org.apache.hadoop.mapred.SortedRanges.Range; + +import org.apache.hadoop.mapreduce.TaskCounter; import org.apache.hadoop.mapreduce.TaskType; import org.apache.hadoop.mapreduce.jobhistory.JobHistory; import org.apache.hadoop.mapreduce.jobhistory.TaskUpdatedEvent; import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo; + +import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; + import org.apache.hadoop.net.Node; + /************************************************************* * TaskInProgress maintains all the info needed for a * Task in the lifetime of its owning Job. A given Task * might be speculatively executed or reexecuted, so we * need a level of indirection above the running-id itself. *
    - * A given TaskInProgress contains multiple taskids, + * A given TaskInProgress contains multiple task attempt ids, * 0 or more of which might be executing at any one time. - * (That's what allows speculative execution.) A taskid - * is now *never* recycled. A TIP allocates enough taskids + * (That's what allows speculative execution.) A task attempt id + * is now *never* recycled. A TIP allocates enough task attempt ids * to account for all the speculation and failures it will * ever have to handle. Once those are up, the TIP is dead. * ************************************************************** @@ -60,6 +67,10 @@ class TaskInProgress { static final long SPECULATIVE_LAG = 60 * 1000; private static final int NUM_ATTEMPTS_PER_RESTART = 1000; + private static final long MEMORY_SPLITS_RESOLUTION = 1024; + + static final int DEFAULT_STATISTICS_INTERVALS = 12; + public static final Log LOG = LogFactory.getLog(TaskInProgress.class); // Defines the TIP @@ -91,6 +102,10 @@ class TaskInProgress { private volatile boolean skipping = false; private boolean jobCleanup = false; private boolean jobSetup = false; + + static final Enum CPU_COUNTER_KEY = TaskCounter.CPU_MILLISECONDS; + static final Enum VM_BYTES_KEY = TaskCounter.VIRTUAL_MEMORY_BYTES; + static final Enum PHYSICAL_BYTES_KEY = TaskCounter.PHYSICAL_MEMORY_BYTES; // The 'next' usable taskid of this tip int nextTaskId = 0; @@ -109,12 +124,20 @@ class TaskInProgress { private JobConf conf; private Map> taskDiagnosticData = new TreeMap>(); + /** - * Map from taskId -> TaskStatus + * Map from task attempt Id -> TaskStatus */ TreeMap taskStatuses = new TreeMap(); + + /** + * Map from task attempt Id -> splits block + */ + private Map splitsBlocks + = new TreeMap(); + // Map from taskId -> TaskTracker Id, // contains cleanup attempts and where they ran, if any private TreeMap cleanupTasks = @@ -183,6 +206,62 @@ class TaskInProgress { } this.user = job.getUser(); } + + synchronized ProgressSplitsBlock getSplits(TaskAttemptID statusAttemptID) { + ProgressSplitsBlock result = splitsBlocks.get(statusAttemptID); + + if (result == null) { + result + = new ProgressSplitsBlock + (conf.getInt(JTConfig.JT_JOBHISTORY_TASKPROGRESS_NUMBER_SPLITS, + ProgressSplitsBlock.DEFAULT_NUMBER_PROGRESS_SPLITS)); + splitsBlocks.put(statusAttemptID, result); + } + + return result; + } + + private void updateProgressSplits(TaskStatus taskStatus) { + double newProgress = taskStatus.getProgress(); + + Counters counters = taskStatus.getCounters(); + if (counters == null) return; + + TaskAttemptID statusAttemptID = taskStatus.getTaskID(); + ProgressSplitsBlock splitsBlock = getSplits(statusAttemptID); + + if (splitsBlock != null) { + + long now = JobTracker.getClock().getTime(); + Long start = getDispatchTime(statusAttemptID); + + if (start != null && now - start <= Integer.MAX_VALUE) { + splitsBlock.progressWallclockTime.extend + (newProgress, (int)(now - start)); + } + + Counters.Counter cpuCounter = counters.findCounter(CPU_COUNTER_KEY); + if (cpuCounter != null + && cpuCounter.getCounter() <= Integer.MAX_VALUE) { + splitsBlock.progressCPUTime.extend + (newProgress, (int)(cpuCounter.getCounter())); + } + + Counters.Counter virtualBytes = counters.findCounter(VM_BYTES_KEY); + if (virtualBytes != null) { + splitsBlock.progressVirtualMemoryKbytes.extend + (newProgress, + (int)(virtualBytes.getCounter() / (MEMORY_SPLITS_RESOLUTION))); + } + + Counters.Counter physicalBytes = counters.findCounter(PHYSICAL_BYTES_KEY); + if (physicalBytes != null) { + splitsBlock.progressPhysicalMemoryKbytes.extend + (newProgress, + (int)(physicalBytes.getCounter() / (MEMORY_SPLITS_RESOLUTION))); + } + } + } /** * Set the max number of attempts before we declare a TIP as "failed" @@ -294,6 +373,7 @@ class TaskInProgress { return execFinishTime; } + /** * Set the exec finish time */ @@ -582,23 +662,24 @@ class TaskInProgress { * @return has the task changed its state noticeably? */ synchronized boolean updateStatus(TaskStatus status) { - TaskAttemptID taskid = status.getTaskID(); - String tracker = status.getTaskTracker(); - String diagInfo = status.getDiagnosticInfo(); - TaskStatus oldStatus = taskStatuses.get(taskid); - boolean changed = true; - if (diagInfo != null && diagInfo.length() > 0) { - LOG.info("Error from " + taskid + " on " + tracker + ": "+ diagInfo); - addDiagnosticInfo(taskid, diagInfo); - } + try { + TaskAttemptID taskid = status.getTaskID(); + String tracker = status.getTaskTracker(); + String diagInfo = status.getDiagnosticInfo(); + TaskStatus oldStatus = taskStatuses.get(taskid); + boolean changed = true; + if (diagInfo != null && diagInfo.length() > 0) { + LOG.info("Error from " + taskid + " on " + tracker + ": "+ diagInfo); + addDiagnosticInfo(taskid, diagInfo); + } - if(skipping) { - failedRanges.updateState(status); - } + if(skipping) { + failedRanges.updateState(status); + } - if (oldStatus != null) { - TaskStatus.State oldState = oldStatus.getRunState(); - TaskStatus.State newState = status.getRunState(); + if (oldStatus != null) { + TaskStatus.State oldState = oldStatus.getRunState(); + TaskStatus.State newState = status.getRunState(); // We should never receive a duplicate success/failure/killed // status update for the same taskid! This is a safety check, @@ -617,60 +698,63 @@ class TaskInProgress { return false; } - // The task is not allowed to move from completed back to running. - // We have seen out of order status messagesmoving tasks from complete - // to running. This is a spot fix, but it should be addressed more - // globally. - if ((newState == TaskStatus.State.RUNNING || - newState == TaskStatus.State.UNASSIGNED) && - (oldState == TaskStatus.State.FAILED || - oldState == TaskStatus.State.KILLED || - oldState == TaskStatus.State.FAILED_UNCLEAN || - oldState == TaskStatus.State.KILLED_UNCLEAN || - oldState == TaskStatus.State.SUCCEEDED || - oldState == TaskStatus.State.COMMIT_PENDING)) { - return false; - } + // The task is not allowed to move from completed back to running. + // We have seen out of order status messagesmoving tasks from complete + // to running. This is a spot fix, but it should be addressed more + // globally. + if ((newState == TaskStatus.State.RUNNING || + newState == TaskStatus.State.UNASSIGNED) && + (oldState == TaskStatus.State.FAILED || + oldState == TaskStatus.State.KILLED || + oldState == TaskStatus.State.FAILED_UNCLEAN || + oldState == TaskStatus.State.KILLED_UNCLEAN || + oldState == TaskStatus.State.SUCCEEDED || + oldState == TaskStatus.State.COMMIT_PENDING)) { + return false; + } - //Do not accept any status once the task is marked FAILED/KILLED - //This is to handle the case of the JobTracker timing out a task - //due to launch delay, but the TT comes back with any state or - //TT got expired - if (oldState == TaskStatus.State.FAILED || - oldState == TaskStatus.State.KILLED) { - tasksToKill.put(taskid, true); - return false; - } + //Do not accept any status once the task is marked FAILED/KILLED + //This is to handle the case of the JobTracker timing out a task + //due to launch delay, but the TT comes back with any state or + //TT got expired + if (oldState == TaskStatus.State.FAILED || + oldState == TaskStatus.State.KILLED) { + tasksToKill.put(taskid, true); + return false; + } - changed = oldState != newState; - } - // if task is a cleanup attempt, do not replace the complete status, - // update only specific fields. - // For example, startTime should not be updated, - // but finishTime has to be updated. - if (!isCleanupAttempt(taskid)) { - taskStatuses.put(taskid, status); - //we don't want to include setup tasks in the task execution stats - if (!isJobSetupTask() && ((isMapTask() && job.hasSpeculativeMaps()) || - (!isMapTask() && job.hasSpeculativeReduces()))) { - long now = JobTracker.getClock().getTime(); - double oldProgRate = getOldProgressRate(); - double currProgRate = getCurrentProgressRate(now); - job.updateStatistics(oldProgRate, currProgRate, isMapTask()); - //we need to store the current progress rate, so that we can - //update statistics accurately the next time we invoke - //updateStatistics - setProgressRate(currProgRate); + changed = oldState != newState; + } + // if task is a cleanup attempt, do not replace the complete status, + // update only specific fields. + // For example, startTime should not be updated, + // but finishTime has to be updated. + if (!isCleanupAttempt(taskid)) { + taskStatuses.put(taskid, status); + //we don't want to include setup tasks in the task execution stats + if (!isJobSetupTask() && ((isMapTask() && job.hasSpeculativeMaps()) || + (!isMapTask() && job.hasSpeculativeReduces()))) { + long now = JobTracker.getClock().getTime(); + double oldProgRate = getOldProgressRate(); + double currProgRate = getCurrentProgressRate(now); + job.updateStatistics(oldProgRate, currProgRate, isMapTask()); + //we need to store the current progress rate, so that we can + //update statistics accurately the next time we invoke + //updateStatistics + setProgressRate(currProgRate); + } + } else { + taskStatuses.get(taskid).statusUpdate(status.getRunState(), + status.getProgress(), status.getStateString(), status.getPhase(), + status.getFinishTime()); } - } else { - taskStatuses.get(taskid).statusUpdate(status.getRunState(), - status.getProgress(), status.getStateString(), status.getPhase(), - status.getFinishTime()); - } - // Recompute progress - recomputeProgress(); - return changed; + // Recompute progress + recomputeProgress(); + return changed; + } finally { + updateProgressSplits(status); + } } /** @@ -953,7 +1037,7 @@ class TaskInProgress { if (status.getProgress() >= bestProgress) { bestProgress = status.getProgress(); bestState = status.getStateString(); - if (status.getIncludeCounters()) { + if (status.getIncludeAllCounters()) { bestCounters = status.getCounters(); } else { bestCounters = this.counters; diff --git a/mapreduce/src/java/org/apache/hadoop/mapred/TaskMemoryManagerThread.java b/mapreduce/src/java/org/apache/hadoop/mapred/TaskMemoryManagerThread.java index bf78fe12dd3..caf6dcbce34 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapred/TaskMemoryManagerThread.java +++ b/mapreduce/src/java/org/apache/hadoop/mapred/TaskMemoryManagerThread.java @@ -227,8 +227,12 @@ class TaskMemoryManagerThread extends Thread { continue; // processTree cannot be tracked } - if (taskTracker.runningTasks.get(tid).wasKilled()) { - continue; // this task has been killed already + TaskInProgress tip = taskTracker.getRunningTask(tid); + if (tip == null) { + continue; + } + if (tip.wasKilled()) { + continue; } LOG.debug("Constructing ProcessTree for : PID = " + pId + " TID = " @@ -514,6 +518,12 @@ class TaskMemoryManagerThread extends Thread { * @param msg diagnostics message */ private void killTask(TaskAttemptID tid, String msg) { + TaskInProgress tip = taskTracker.getRunningTask(tid); + if (tip != null) { + //for the task identified to be killed update taskDiagnostic + TaskStatus taskStatus = tip.getStatus(); + taskStatus.setDiagnosticInfo(msg); + } // Kill the task and mark it as killed. taskTracker.cleanUpOverMemoryTask(tid, false, msg); // Now destroy the ProcessTree, remove it from monitoring map. @@ -530,7 +540,7 @@ class TaskMemoryManagerThread extends Thread { * @return true if the task can be killed */ private boolean isKillable(TaskAttemptID tid) { - TaskInProgress tip = taskTracker.runningTasks.get(tid); + TaskInProgress tip = taskTracker.getRunningTask(tid); return tip != null && !tip.wasKilled() && (tip.getRunState() == TaskStatus.State.RUNNING || tip.getRunState() == TaskStatus.State.COMMIT_PENDING); diff --git a/mapreduce/src/java/org/apache/hadoop/mapred/TaskStatus.java b/mapreduce/src/java/org/apache/hadoop/mapred/TaskStatus.java index 330dbc6768d..9b29bba65fc 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapred/TaskStatus.java +++ b/mapreduce/src/java/org/apache/hadoop/mapred/TaskStatus.java @@ -66,7 +66,7 @@ public abstract class TaskStatus implements Writable, Cloneable { private volatile Phase phase = Phase.STARTING; private Counters counters; - private boolean includeCounters; + private boolean includeAllCounters; private SortedRanges.Range nextRecordRange = new SortedRanges.Range(); // max task-status string size @@ -100,7 +100,7 @@ public abstract class TaskStatus implements Writable, Cloneable { this.taskTracker = taskTracker; this.phase = phase; this.counters = counters; - this.includeCounters = true; + this.includeAllCounters = true; } public TaskAttemptID getTaskID() { return taskid; } @@ -311,12 +311,13 @@ public abstract class TaskStatus implements Writable, Cloneable { this.runState == TaskStatus.State.KILLED_UNCLEAN)); } - public boolean getIncludeCounters() { - return includeCounters; + public boolean getIncludeAllCounters() { + return includeAllCounters; } - public void setIncludeCounters(boolean send) { - includeCounters = send; + public void setIncludeAllCounters(boolean send) { + includeAllCounters = send; + counters.setWriteAllCounters(send); } /** @@ -465,11 +466,9 @@ public abstract class TaskStatus implements Writable, Cloneable { WritableUtils.writeEnum(out, phase); out.writeLong(startTime); out.writeLong(finishTime); - out.writeBoolean(includeCounters); + out.writeBoolean(includeAllCounters); out.writeLong(outputSize); - if (includeCounters) { - counters.write(out); - } + counters.write(out); nextRecordRange.write(out); } @@ -484,11 +483,9 @@ public abstract class TaskStatus implements Writable, Cloneable { this.startTime = in.readLong(); this.finishTime = in.readLong(); counters = new Counters(); - this.includeCounters = in.readBoolean(); + this.includeAllCounters = in.readBoolean(); this.outputSize = in.readLong(); - if (includeCounters) { - counters.readFields(in); - } + counters.readFields(in); nextRecordRange.readFields(in); } diff --git a/mapreduce/src/java/org/apache/hadoop/mapred/TaskTracker.java b/mapreduce/src/java/org/apache/hadoop/mapred/TaskTracker.java index 52ea9162b19..20e63faa2ce 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapred/TaskTracker.java +++ b/mapreduce/src/java/org/apache/hadoop/mapred/TaskTracker.java @@ -1617,13 +1617,13 @@ public class TaskTracker */ HeartbeatResponse transmitHeartBeat(long now) throws IOException { // Send Counters in the status once every COUNTER_UPDATE_INTERVAL - boolean sendCounters; + boolean sendAllCounters; if (now > (previousUpdate + COUNTER_UPDATE_INTERVAL)) { - sendCounters = true; + sendAllCounters = true; previousUpdate = now; } else { - sendCounters = false; + sendAllCounters = false; } // @@ -1636,7 +1636,7 @@ public class TaskTracker status = new TaskTrackerStatus(taskTrackerName, localHostname, httpPort, cloneAndResetRunningTaskStatuses( - sendCounters), + sendAllCounters), failures, maxMapSlots, maxReduceSlots); @@ -3521,10 +3521,10 @@ public class TaskTracker List result = new ArrayList(runningTasks.size()); for(TaskInProgress tip: runningTasks.values()) { TaskStatus status = tip.getStatus(); - status.setIncludeCounters(sendCounters); + status.setIncludeAllCounters(sendCounters); // send counters for finished or failed tasks and commit pending tasks if (status.getRunState() != TaskStatus.State.RUNNING) { - status.setIncludeCounters(true); + status.setIncludeAllCounters(true); } result.add((TaskStatus)status.clone()); status.clearStatus(); @@ -4218,4 +4218,8 @@ public class TaskTracker ACLsManager getACLsManager() { return aclsManager; } + + synchronized TaskInProgress getRunningTask(TaskAttemptID tid) { + return runningTasks.get(tid); + } } diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/Counter.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/Counter.java index 09b0319a1f9..b74bbf32325 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapreduce/Counter.java +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/Counter.java @@ -18,137 +18,58 @@ package org.apache.hadoop.mapreduce; -import java.io.IOException; -import java.io.DataInput; -import java.io.DataOutput; - import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableUtils; /** * A named counter that tracks the progress of a map/reduce job. - * - *

    Counters represent global counters, defined either by the + * + *

    Counters represent global counters, defined either by the * Map-Reduce framework or applications. Each Counter is named by * an {@link Enum} and has a long for the value.

    - * + * *

    Counters are bunched into Groups, each comprising of - * counters from a particular Enum class. + * counters from a particular Enum class. */ @InterfaceAudience.Public @InterfaceStability.Stable -public class Counter implements Writable { +public interface Counter extends Writable { - private String name; - private String displayName; - private long value = 0; - - protected Counter() { - } - - protected Counter(String name, String displayName) { - this.name = name; - this.displayName = displayName; - } - - /** Create a counter. - * @param name the name within the group's enum. - * @param displayName a name to be displayed. - * @param value the counter value. + /** + * Set the display name of the counter + * @param displayName of the counter + * @deprecated (and no-op by default) */ - public Counter(String name, String displayName, long value) { - this.name = name; - this.displayName = displayName; - this.value = value; - } - @Deprecated - protected synchronized void setDisplayName(String displayName) { - this.displayName = displayName; - } - - /** - * Read the binary representation of the counter - */ - @Override - public synchronized void readFields(DataInput in) throws IOException { - name = Text.readString(in); - if (in.readBoolean()) { - displayName = Text.readString(in); - } else { - displayName = name; - } - value = WritableUtils.readVLong(in); - } - - /** - * Write the binary representation of the counter - */ - @Override - public synchronized void write(DataOutput out) throws IOException { - Text.writeString(out, name); - boolean distinctDisplayName = ! name.equals(displayName); - out.writeBoolean(distinctDisplayName); - if (distinctDisplayName) { - Text.writeString(out, displayName); - } - WritableUtils.writeVLong(out, value); - } - - public synchronized String getName() { - return name; - } + void setDisplayName(String displayName); /** - * Get the name of the counter. + * @return the name of the counter + */ + String getName(); + + /** + * Get the display name of the counter. * @return the user facing name of the counter */ - public synchronized String getDisplayName() { - return displayName; - } - + String getDisplayName(); + /** * What is the current value of this counter? * @return the current value */ - public synchronized long getValue() { - return value; - } + long getValue(); /** * Set this counter by the given value * @param value the value to set */ - public synchronized void setValue(long value) { - this.value = value; - } + void setValue(long value); /** * Increment this counter by the given value * @param incr the value to increase this counter by */ - public synchronized void increment(long incr) { - value += incr; - } - - @Override - public synchronized boolean equals(Object genericRight) { - if (genericRight instanceof Counter) { - synchronized (genericRight) { - Counter right = (Counter) genericRight; - return name.equals(right.name) && - displayName.equals(right.displayName) && - value == right.value; - } - } - return false; - } - - @Override - public synchronized int hashCode() { - return name.hashCode() + displayName.hashCode(); - } + void increment(long incr); } diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/CounterGroup.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/CounterGroup.java index 57a5525e470..f81d947c654 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapreduce/CounterGroup.java +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/CounterGroup.java @@ -18,19 +18,9 @@ package org.apache.hadoop.mapreduce; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.Iterator; -import java.util.MissingResourceException; -import java.util.ResourceBundle; -import java.util.TreeMap; - import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableUtils; +import org.apache.hadoop.mapreduce.counters.CounterGroupBase; /** * A group of {@link Counter}s that logically belong together. Typically, @@ -38,156 +28,6 @@ import org.apache.hadoop.io.WritableUtils; */ @InterfaceAudience.Public @InterfaceStability.Stable -public class CounterGroup implements Writable, Iterable { - private String name; - private String displayName; - private TreeMap counters = new TreeMap(); - // Optional ResourceBundle for localization of group and counter names. - private ResourceBundle bundle = null; - - /** - * Returns the specified resource bundle, or throws an exception. - * @throws MissingResourceException if the bundle isn't found - */ - private static ResourceBundle getResourceBundle(String enumClassName) { - String bundleName = enumClassName.replace('$','_'); - return ResourceBundle.getBundle(bundleName); - } - - protected CounterGroup(String name) { - this.name = name; - try { - bundle = getResourceBundle(name); - } - catch (MissingResourceException neverMind) { - } - displayName = localize("CounterGroupName", name); - } - - /** Create a CounterGroup. - * @param name the name of the group's enum. - * @param displayName a name to be displayed for the group. - */ - public CounterGroup(String name, String displayName) { - this.name = name; - this.displayName = displayName; - } - - /** - * Get the internal name of the group - * @return the internal name - */ - public synchronized String getName() { - return name; - } - - /** - * Get the display name of the group. - * @return the human readable name - */ - public synchronized String getDisplayName() { - return displayName; - } - - /** Add a counter to this group. */ - public synchronized void addCounter(Counter counter) { - counters.put(counter.getName(), counter); - } - - /** - * Find a counter in a group. - * @param counterName the name of the counter - * @param displayName the display name of the counter - * @return the counter that was found or added - */ - public Counter findCounter(String counterName, String displayName) { - Counter result = counters.get(counterName); - if (result == null) { - result = new Counter(counterName, displayName); - counters.put(counterName, result); - } - return result; - } - - public synchronized Counter findCounter(String counterName) { - Counter result = counters.get(counterName); - if (result == null) { - String displayName = localize(counterName, counterName); - result = new Counter(counterName, displayName); - counters.put(counterName, result); - } - return result; - } - - public synchronized Iterator iterator() { - return counters.values().iterator(); - } - - public synchronized void write(DataOutput out) throws IOException { - Text.writeString(out, displayName); - WritableUtils.writeVInt(out, counters.size()); - for(Counter counter: counters.values()) { - counter.write(out); - } - } - - public synchronized void readFields(DataInput in) throws IOException { - displayName = Text.readString(in); - counters.clear(); - int size = WritableUtils.readVInt(in); - for(int i=0; i < size; i++) { - Counter counter = new Counter(); - counter.readFields(in); - counters.put(counter.getName(), counter); - } - } - - /** - * Looks up key in the ResourceBundle and returns the corresponding value. - * If the bundle or the key doesn't exist, returns the default value. - */ - private String localize(String key, String defaultValue) { - String result = defaultValue; - if (bundle != null) { - try { - result = bundle.getString(key); - } - catch (MissingResourceException mre) { - } - } - return result; - } - - /** - * Returns the number of counters in this group. - */ - public synchronized int size() { - return counters.size(); - } - - public synchronized boolean equals(Object genericRight) { - if (genericRight instanceof CounterGroup) { - Iterator right = ((CounterGroup) genericRight).counters. - values().iterator(); - Iterator left = counters.values().iterator(); - while (left.hasNext()) { - if (!right.hasNext() || !left.next().equals(right.next())) { - return false; - } - } - return !right.hasNext(); - } - return false; - } - - public synchronized int hashCode() { - return counters.hashCode(); - } - - public synchronized void incrAllCounters(CounterGroup rightGroup) { - for(Counter right: rightGroup.counters.values()) { - Counter left = findCounter(right.getName(), right.getDisplayName()); - left.increment(right.getValue()); - } - } +public interface CounterGroup extends CounterGroupBase { + // essentially a typedef so user doesn't have to use generic syntax } diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/Counters.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/Counters.java index 4896b350027..8f82523adf5 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapreduce/Counters.java +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/Counters.java @@ -17,200 +17,121 @@ */ package org.apache.hadoop.mapreduce; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.Collection; -import java.util.IdentityHashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.TreeMap; - import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapreduce.counters.Limits; +import org.apache.hadoop.mapreduce.counters.GenericCounter; +import org.apache.hadoop.mapreduce.counters.AbstractCounterGroup; +import org.apache.hadoop.mapreduce.counters.CounterGroupBase; +import org.apache.hadoop.mapreduce.counters.FileSystemCounterGroup; +import org.apache.hadoop.mapreduce.counters.AbstractCounters; +import org.apache.hadoop.mapreduce.counters.CounterGroupFactory; +import org.apache.hadoop.mapreduce.counters.FrameworkCounterGroup; +/** + *

    Counters holds per job/task counters, defined either by the + * Map-Reduce framework or applications. Each Counter can be of + * any {@link Enum} type.

    + * + *

    Counters are bunched into {@link CounterGroup}s, each + * comprising of counters from a particular Enum class. + */ @InterfaceAudience.Public @InterfaceStability.Stable -public class Counters implements Writable,Iterable { - /** - * A cache from enum values to the associated counter. Dramatically speeds up - * typical usage. - */ - private Map, Counter> cache = new IdentityHashMap, Counter>(); +public class Counters extends AbstractCounters { - private TreeMap groups = - new TreeMap(); - - public Counters() { - } - - /** - * Utility method to create a Counters object from the - * org.apache.hadoop.mapred counters - * @param counters - */ - public Counters(org.apache.hadoop.mapred.Counters counters) { - for(org.apache.hadoop.mapred.Counters.Group group: counters) { - String name = group.getName(); - CounterGroup newGroup = new CounterGroup(name, group.getDisplayName()); - groups.put(name, newGroup); - for(Counter counter: group) { - newGroup.addCounter(counter); - } + // Mix framework group implementation into CounterGroup interface + private static class FrameworkGroupImpl> + extends FrameworkCounterGroup implements CounterGroup { + + FrameworkGroupImpl(Class cls) { + super(cls); + } + + @Override + protected FrameworkCounter newCounter(T key) { + return new FrameworkCounter(key); } } - /** Add a group. */ - public void addGroup(CounterGroup group) { - groups.put(group.getName(), group); - } + // Mix generic group implementation into CounterGroup interface + // and provide some mandatory group factory methods. + private static class GenericGroup extends AbstractCounterGroup + implements CounterGroup { - public Counter findCounter(String groupName, String counterName) { - CounterGroup grp = getGroup(groupName); - return grp.findCounter(counterName); - } - - /** - * Find the counter for the given enum. The same enum will always return the - * same counter. - * @param key the counter key - * @return the matching counter object - */ - public synchronized Counter findCounter(Enum key) { - Counter counter = cache.get(key); - if (counter == null) { - counter = findCounter(key.getDeclaringClass().getName(), key.toString()); - cache.put(key, counter); + GenericGroup(String name, String displayName, Limits limits) { + super(name, displayName, limits); } - return counter; - } - /** - * Returns the names of all counter classes. - * @return Set of counter names. - */ - public synchronized Collection getGroupNames() { - return groups.keySet(); - } - - @Override - public Iterator iterator() { - return groups.values().iterator(); - } - - /** - * Returns the named counter group, or an empty group if there is none - * with the specified name. - */ - public synchronized CounterGroup getGroup(String groupName) { - CounterGroup grp = groups.get(groupName); - if (grp == null) { - grp = new CounterGroup(groupName); - groups.put(groupName, grp); + @Override + protected Counter newCounter(String name, String displayName, long value) { + return new GenericCounter(name, displayName, value); } - return grp; - } - /** - * Returns the total number of counters, by summing the number of counters - * in each group. - */ - public synchronized int countCounters() { - int result = 0; - for (CounterGroup group : this) { - result += group.size(); - } - return result; - } - - /** - * Write the set of groups. - * The external format is: - * #groups (groupName group)* - * - * i.e. the number of groups followed by 0 or more groups, where each - * group is of the form: - * - * groupDisplayName #counters (false | true counter)* - * - * where each counter is of the form: - * - * name (false | true displayName) value - */ - @Override - public synchronized void write(DataOutput out) throws IOException { - out.writeInt(groups.size()); - for (org.apache.hadoop.mapreduce.CounterGroup group: groups.values()) { - Text.writeString(out, group.getName()); - group.write(out); + @Override + protected Counter newCounter() { + return new GenericCounter(); } } - - /** - * Read a set of groups. - */ - @Override - public synchronized void readFields(DataInput in) throws IOException { - int numClasses = in.readInt(); - groups.clear(); - while (numClasses-- > 0) { - String groupName = Text.readString(in); - CounterGroup group = new CounterGroup(groupName); - group.readFields(in); - groups.put(groupName, group); + + // Mix file system group implementation into the CounterGroup interface + private static class FileSystemGroup extends FileSystemCounterGroup + implements CounterGroup { + + @Override + protected Counter newCounter(String scheme, FileSystemCounter key) { + return new FSCounter(scheme, key); } } /** - * Return textual representation of the counter values. + * Provide factory methods for counter group factory implementation. + * See also the GroupFactory in + * {@link org.apache.hadoop.mapred.Counters mapred.Counters} */ - public synchronized String toString() { - StringBuilder sb = new StringBuilder("Counters: " + countCounters()); - for (CounterGroup group: this) { - sb.append("\n\t" + group.getDisplayName()); - for (Counter counter: group) { - sb.append("\n\t\t" + counter.getDisplayName() + "=" + - counter.getValue()); - } - } - return sb.toString(); - } + private static class GroupFactory + extends CounterGroupFactory { - /** - * Increments multiple counters by their amounts in another Counters - * instance. - * @param other the other Counters instance - */ - public synchronized void incrAllCounters(Counters other) { - for(Map.Entry rightEntry: other.groups.entrySet()) { - CounterGroup left = groups.get(rightEntry.getKey()); - CounterGroup right = rightEntry.getValue(); - if (left == null) { - left = new CounterGroup(right.getName(), right.getDisplayName()); - groups.put(rightEntry.getKey(), left); - } - left.incrAllCounters(right); - } - } - - public boolean equals(Object genericRight) { - if (genericRight instanceof Counters) { - Iterator right = ((Counters) genericRight).groups. - values().iterator(); - Iterator left = groups.values().iterator(); - while (left.hasNext()) { - if (!right.hasNext() || !left.next().equals(right.next())) { - return false; + @Override + protected > + FrameworkGroupFactory + newFrameworkGroupFactory(final Class cls) { + return new FrameworkGroupFactory() { + @Override public CounterGroup newGroup(String name) { + return new FrameworkGroupImpl(cls); // impl in this package } - } - return !right.hasNext(); + }; + } + + @Override + protected CounterGroup newGenericGroup(String name, String displayName, + Limits limits) { + return new GenericGroup(name, displayName, limits); + } + + @Override + protected CounterGroup newFileSystemGroup() { + return new FileSystemGroup(); } - return false; } - - public int hashCode() { - return groups.hashCode(); + + private static final GroupFactory groupFactory = new GroupFactory(); + + /** + * Default constructor + */ + public Counters() { + super(groupFactory); + } + + /** + * Construct the Counters object from the another counters object + * @param the type of counter + * @param the type of counter group + * @param counters the old counters object + */ + public > + Counters(AbstractCounters counters) { + super(counters, groupFactory); } } diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/FileSystemCounter.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/FileSystemCounter.java new file mode 100644 index 00000000000..3624b1a99c7 --- /dev/null +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/FileSystemCounter.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce; + +import org.apache.hadoop.classification.InterfaceAudience; + +@InterfaceAudience.Private +public enum FileSystemCounter { + BYTES_READ, + BYTES_WRITTEN, + READ_OPS, + LARGE_READ_OPS, + WRITE_OPS, +} diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/FileSystemCounter.properties b/mapreduce/src/java/org/apache/hadoop/mapreduce/FileSystemCounter.properties new file mode 100644 index 00000000000..58089af8039 --- /dev/null +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/FileSystemCounter.properties @@ -0,0 +1,21 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ResourceBundle properties file for job-level counters + +CounterGroupName= File System Counters + +BYTES_READ.name= Number of bytes read +BYTES_WRITTEN.name= Number of bytes written +READ_OPS.name= Number of read operations +LARGE_READ_OPS.name= Number of large read operations +WRITE_OPS.name= Number of write operations diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/Job.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/Job.java index 0be084077d4..96c2bc53a19 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapreduce/Job.java +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/Job.java @@ -122,7 +122,7 @@ public class Job extends JobContextImpl implements JobContext { private JobStatus status; private long statustime; private Cluster cluster; - + @Deprecated public Job() throws IOException { this(new Configuration()); @@ -360,8 +360,11 @@ public class Job extends JobContextImpl implements JobContext { @Override public String toString() { ensureState(JobState.RUNNING); + String reasonforFailure = " "; try { updateStatus(); + if (status.getState().equals(JobStatus.State.FAILED)) + reasonforFailure = getTaskFailureEventString(); } catch (IOException e) { } catch (InterruptedException ie) { } @@ -378,10 +381,34 @@ public class Job extends JobContextImpl implements JobContext { sb.append(status.getState()).append("\n"); sb.append("history URL: "); sb.append(status.getHistoryFile()).append("\n"); - sb.append("retired: ").append(status.isRetired()); + sb.append("retired: ").append(status.isRetired()).append("\n"); + sb.append("reason for failure: ").append(reasonforFailure); return sb.toString(); } - + + /** + * @return taskid which caused job failure + * @throws IOException + * @throws InterruptedException + */ + String getTaskFailureEventString() throws IOException, + InterruptedException { + int failCount = 1; + TaskCompletionEvent lastEvent = null; + for (TaskCompletionEvent event : cluster.getClient().getTaskCompletionEvents( + status.getJobID(), 0, 10)) { + if (event.getStatus().equals(TaskCompletionEvent.Status.FAILED)) { + failCount++; + lastEvent = event; + } + } + String[] taskAttemptID = lastEvent.getTaskAttemptId().toString().split("_", 2); + String taskID = taskAttemptID[1].substring(0, taskAttemptID[1].length()-2); + return (" task " + taskID + " failed " + + failCount + " times " + "For details check tasktracker at: " + + lastEvent.getTaskTrackerHttp()); + } + /** * Get the information of the current state of the tasks of a job. * diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/JobCounter.properties b/mapreduce/src/java/org/apache/hadoop/mapreduce/JobCounter.properties index 7eaf74426a9..98e9279958f 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapreduce/JobCounter.properties +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/JobCounter.properties @@ -21,5 +21,7 @@ TOTAL_LAUNCHED_REDUCES.name= Launched reduce tasks OTHER_LOCAL_MAPS.name= Other local map tasks DATA_LOCAL_MAPS.name= Data-local map tasks RACK_LOCAL_MAPS.name= Rack-local map tasks +SLOTS_MILLIS_MAPS.name= Total time spent by all maps in occupied slots (ms) +SLOTS_MILLIS_REDUCES.name= Total time spent by all reduces in occupied slots (ms) FALLOW_SLOTS_MILLIS_MAPS.name= Total time spent by all maps waiting after reserving slots (ms) FALLOW_SLOTS_MILLIS_REDUCES.name= Total time spent by all reduces waiting after reserving slots (ms) diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/MRConfig.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/MRConfig.java index 9e88ea182ea..6f4a1628739 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapreduce/MRConfig.java +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/MRConfig.java @@ -59,4 +59,6 @@ public interface MRConfig { 7*24*60*60*1000; // 7 days public static final String FRAMEWORK_NAME = "mapreduce.framework.name"; + public static final String TASK_LOCAL_OUTPUT_CLASS = + "mapreduce.task.local.output.class"; } diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 0054646caf1..c3c8349bd96 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -275,4 +275,16 @@ public interface MRJobConfig { "mapreduce.job.submithostname"; public static final String JOB_SUBMITHOSTADDR = "mapreduce.job.submithostaddress"; + + public static final String COUNTERS_MAX_KEY = "mapreduce.job.counters.max"; + public static final int COUNTERS_MAX_DEFAULT = 120; + + public static final String COUNTER_GROUP_NAME_MAX_KEY = "mapreduce.job.counters.group.name.max"; + public static final int COUNTER_GROUP_NAME_MAX_DEFAULT = 128; + + public static final String COUNTER_NAME_MAX_KEY = "mapreduce.job.counters.counter.name.max"; + public static final int COUNTER_NAME_MAX_DEFAULT = 64; + + public static final String COUNTER_GROUPS_MAX_KEY = "mapreduce.job.counters.groups.max"; + public static final int COUNTER_GROUPS_MAX_DEFAULT = 50; } diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/TaskCounter.properties b/mapreduce/src/java/org/apache/hadoop/mapreduce/TaskCounter.properties index 9cfdea6e937..d54b980b241 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapreduce/TaskCounter.properties +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/TaskCounter.properties @@ -27,9 +27,13 @@ REDUCE_INPUT_RECORDS.name= Reduce input records REDUCE_OUTPUT_RECORDS.name= Reduce output records REDUCE_SKIPPED_RECORDS.name= Reduce skipped records REDUCE_SKIPPED_GROUPS.name= Reduce skipped groups +SPLIT_RAW_BYTES.name= Input split bytes SPILLED_RECORDS.name= Spilled Records SHUFFLED_MAPS.name= Shuffled Maps FAILED_SHUFFLE.name= Failed Shuffles MERGED_MAP_OUTPUTS.name= Merged Map outputs GC_TIME_MILLIS.name= GC time elapsed (ms) COMMITTED_HEAP_BYTES.name= Total committed heap usage (bytes) +CPU_MILLISECONDS.name= CPU time spent (ms) +PHYSICAL_MEMORY_BYTES.name= Physical memory (bytes) snapshot +VIRTUAL_MEMORY_BYTES.name= Virtual memory (bytes) snapshot diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/AbstractCounter.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/AbstractCounter.java new file mode 100644 index 00000000000..dc026cad5ba --- /dev/null +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/AbstractCounter.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce.counters; + +import com.google.common.base.Objects; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.mapreduce.Counter; + +/** + * An abstract counter class to provide common implementation of + * the counter interface in both mapred and mapreduce packages. + */ +@InterfaceAudience.Private +public abstract class AbstractCounter implements Counter { + + @Override @Deprecated + public void setDisplayName(String name) {} + + @Override + public synchronized boolean equals(Object genericRight) { + if (genericRight instanceof Counter) { + synchronized (genericRight) { + Counter right = (Counter) genericRight; + return getName().equals(right.getName()) && + getDisplayName().equals(right.getDisplayName()) && + getValue() == right.getValue(); + } + } + return false; + } + + @Override + public synchronized int hashCode() { + return Objects.hashCode(getName(), getDisplayName(), getValue()); + } +} diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/AbstractCounterGroup.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/AbstractCounterGroup.java new file mode 100644 index 00000000000..de6c4f20c49 --- /dev/null +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/AbstractCounterGroup.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce.counters; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Iterator; +import java.util.Map; + +import com.google.common.collect.Iterators; +import com.google.common.collect.Maps; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.WritableUtils; +import org.apache.hadoop.mapreduce.Counter; +import org.apache.hadoop.mapreduce.util.ResourceBundles; + +/** + * An abstract class to provide common implementation of the + * generic counter group in both mapred and mapreduce package. + * + * @param type of the counter for the group + */ +@InterfaceAudience.Private +public abstract class AbstractCounterGroup + implements CounterGroupBase { + + private final String name; + private String displayName; + private final Map counters = Maps.newTreeMap(); + private final Limits limits; + + public AbstractCounterGroup(String name, String displayName, + Limits limits) { + this.name = name; + this.displayName = displayName; + this.limits = limits; + } + + @Override + public synchronized String getName() { + return name; + } + + @Override + public synchronized String getDisplayName() { + return displayName; + } + + @Override + public synchronized void setDisplayName(String displayName) { + this.displayName = displayName; + } + + @Override + public synchronized void addCounter(T counter) { + counters.put(counter.getName(), counter); + limits.incrCounters(); + } + + @Override + public synchronized T addCounter(String counterName, String displayName, + long value) { + String saveName = limits.filterCounterName(counterName); + T counter = findCounterImpl(saveName, false); + if (counter == null) { + return addCounterImpl(saveName, displayName, value); + } + counter.setValue(value); + return counter; + } + + private T addCounterImpl(String name, String displayName, long value) { + T counter = newCounter(name, displayName, value); + addCounter(counter); + return counter; + } + + @Override + public T findCounter(String counterName, String displayName) { + String saveName = limits.filterCounterName(counterName); + T counter = findCounterImpl(saveName, false); + if (counter == null) { + return addCounterImpl(saveName, displayName, 0); + } + return counter; + } + + @Override + public synchronized T findCounter(String counterName, boolean create) { + return findCounterImpl(limits.filterCounterName(counterName), create); + } + + private T findCounterImpl(String counterName, boolean create) { + T counter = counters.get(counterName); + if (counter == null && create) { + String localized = + ResourceBundles.getCounterName(getName(), counterName, counterName); + return addCounterImpl(counterName, localized, 0); + } + return counter; + } + + @Override + public T findCounter(String counterName) { + return findCounter(counterName, true); + } + + /** + * Abstract factory method to create a new counter of type T + * @param counterName of the counter + * @param displayName of the counter + * @param value of the counter + * @return a new counter + */ + protected abstract T newCounter(String counterName, String displayName, + long value); + + /** + * Abstract factory method to create a new counter of type T + * @return a new counter object + */ + protected abstract T newCounter(); + + @Override + public synchronized Iterator iterator() { + return counters.values().iterator(); + } + + /** + * GenericGroup ::= displayName #counter counter* + */ + @Override + public synchronized void write(DataOutput out) throws IOException { + Text.writeString(out, displayName); + WritableUtils.writeVInt(out, counters.size()); + for(Counter counter: counters.values()) { + counter.write(out); + } + } + + @Override + public synchronized void readFields(DataInput in) throws IOException { + displayName = Text.readString(in); + counters.clear(); + int size = WritableUtils.readVInt(in); + for (int i = 0; i < size; i++) { + T counter = newCounter(); + counter.readFields(in); + counters.put(counter.getName(), counter); + limits.incrCounters(); + } + } + + @Override + public synchronized int size() { + return counters.size(); + } + + @Override + public synchronized boolean equals(Object genericRight) { + if (genericRight instanceof CounterGroupBase) { + @SuppressWarnings("unchecked") + CounterGroupBase right = (CounterGroupBase) genericRight; + return Iterators.elementsEqual(iterator(), right.iterator()); + } + return false; + } + + @Override + public synchronized int hashCode() { + return counters.hashCode(); + } + + @Override + public void incrAllCounters(CounterGroupBase rightGroup) { + try { + for (Counter right : rightGroup) { + Counter left = findCounter(right.getName(), right.getDisplayName()); + left.increment(right.getValue()); + } + } catch (LimitExceededException e) { + counters.clear(); + throw e; + } + } +} diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java new file mode 100644 index 00000000000..7102467a106 --- /dev/null +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java @@ -0,0 +1,371 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce.counters; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Iterator; +import java.util.Map; + +import com.google.common.collect.Iterables; +import com.google.common.collect.Iterators; +import com.google.common.collect.Maps; + +import org.apache.commons.logging.LogFactory; +import org.apache.commons.logging.Log; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableUtils; +import org.apache.hadoop.mapreduce.Counter; +import org.apache.hadoop.mapreduce.FileSystemCounter; +import org.apache.hadoop.mapreduce.JobCounter; +import org.apache.hadoop.mapreduce.TaskCounter; +import static org.apache.hadoop.mapreduce.counters.CounterGroupFactory.*; + +/** + * An abstract class to provide common implementation for the Counters + * container in both mapred and mapreduce packages. + * + * @param type of counter inside the counters + * @param type of group inside the counters + */ +@InterfaceAudience.Public +@InterfaceStability.Stable +public abstract class AbstractCounters> + implements Writable, Iterable { + + protected static final Log LOG = LogFactory.getLog("mapreduce.Counters"); + + /** + * A cache from enum values to the associated counter. + */ + private Map, C> cache = Maps.newIdentityHashMap(); + private Map fgroups = Maps.newTreeMap(); // framework & fs groups + private Map groups = Maps.newTreeMap(); // other groups + private final CounterGroupFactory groupFactory; + + // For framework counter serialization without strings + enum GroupType { FRAMEWORK, FILESYSTEM }; + + // Writes only framework and fs counters if false. + private boolean writeAllCounters = true; + + private static final Map legacyMap = Maps.newHashMap(); + static { + legacyMap.put("org.apache.hadoop.mapred.Task$Counter", + TaskCounter.class.getName()); + legacyMap.put("org.apache.hadoop.mapred.JobInProgress$Counter", + JobCounter.class.getName()); + } + + private final Limits limits = new Limits(); + + @InterfaceAudience.Private + public AbstractCounters(CounterGroupFactory gf) { + groupFactory = gf; + } + + /** + * Construct from another counters object. + * @param type of the other counter + * @param type of the other counter group + * @param counters the counters object to copy + * @param groupFactory the factory for new groups + */ + @InterfaceAudience.Private + public > + AbstractCounters(AbstractCounters counters, + CounterGroupFactory groupFactory) { + this.groupFactory = groupFactory; + for(G1 group: counters) { + String name = group.getName(); + G newGroup = groupFactory.newGroup(name, group.getDisplayName(), limits); + (isFrameworkGroup(name) ? fgroups : groups).put(name, newGroup); + for(Counter counter: group) { + newGroup.addCounter(counter.getName(), counter.getDisplayName(), + counter.getValue()); + } + } + } + + /** Add a group. + * @param group object to add + * @return the group + */ + @InterfaceAudience.Private + public synchronized G addGroup(G group) { + String name = group.getName(); + if (isFrameworkGroup(name)) { + fgroups.put(name, group); + } else { + limits.checkGroups(groups.size() + 1); + groups.put(name, group); + } + return group; + } + + /** + * Add a new group + * @param name of the group + * @param displayName of the group + * @return the group + */ + @InterfaceAudience.Private + public G addGroup(String name, String displayName) { + return addGroup(groupFactory.newGroup(name, displayName, limits)); + } + + /** + * Find a counter, create one if necessary + * @param groupName of the counter + * @param counterName name of the counter + * @return the matching counter + */ + public C findCounter(String groupName, String counterName) { + G grp = getGroup(groupName); + return grp.findCounter(counterName); + } + + /** + * Find the counter for the given enum. The same enum will always return the + * same counter. + * @param key the counter key + * @return the matching counter object + */ + public synchronized C findCounter(Enum key) { + C counter = cache.get(key); + if (counter == null) { + counter = findCounter(key.getDeclaringClass().getName(), key.name()); + cache.put(key, counter); + } + return counter; + } + + /** + * Find the file system counter for the given scheme and enum. + * @param scheme of the file system + * @param key the enum of the counter + * @return the file system counter + */ + @InterfaceAudience.Private + public synchronized C findCounter(String scheme, FileSystemCounter key) { + return ((FileSystemCounterGroup) getGroup( + FileSystemCounter.class.getName())).findCounter(scheme, key); + } + + /** + * Returns the names of all counter classes. + * @return Set of counter names. + */ + public synchronized Iterable getGroupNames() { + return Iterables.concat(fgroups.keySet(), groups.keySet()); + } + + @Override + public Iterator iterator() { + return Iterators.concat(fgroups.values().iterator(), + groups.values().iterator()); + } + + /** + * Returns the named counter group, or an empty group if there is none + * with the specified name. + * @param groupName name of the group + * @return the group + */ + public synchronized G getGroup(String groupName) { + boolean isFGroup = isFrameworkGroup(groupName); + G group = isFGroup ? fgroups.get(groupName) : groups.get(groupName); + if (group == null) { + group = groupFactory.newGroup(filterGroupName(groupName), limits); + if (isFGroup) { + fgroups.put(groupName, group); + } else { + limits.checkGroups(groups.size() + 1); + groups.put(groupName, group); + } + } + return group; + } + + private String filterGroupName(String oldName) { + String newName = legacyMap.get(oldName); + if (newName == null) { + return limits.filterGroupName(oldName); + } + LOG.warn("Group "+ oldName +" is deprecated. Use "+ newName +" instead"); + return newName; + } + + /** + * Returns the total number of counters, by summing the number of counters + * in each group. + * @return the total number of counters + */ + public synchronized int countCounters() { + int result = 0; + for (G group : this) { + result += group.size(); + } + return result; + } + + /** + * Write the set of groups. + * Counters ::= version #fgroups (groupId, group)* #groups (group)* + */ + @Override + public synchronized void write(DataOutput out) throws IOException { + WritableUtils.writeVInt(out, groupFactory.version()); + WritableUtils.writeVInt(out, fgroups.size()); // framework groups first + for (G group : fgroups.values()) { + if (group instanceof FrameworkCounterGroup) { + WritableUtils.writeVInt(out, GroupType.FRAMEWORK.ordinal()); + WritableUtils.writeVInt(out, getFrameworkGroupId(group.getName())); + group.write(out); + } else if (group instanceof FileSystemCounterGroup) { + WritableUtils.writeVInt(out, GroupType.FILESYSTEM.ordinal()); + group.write(out); + } + } + if (writeAllCounters) { + WritableUtils.writeVInt(out, groups.size()); + for (G group : groups.values()) { + Text.writeString(out, group.getName()); + group.write(out); + } + } else { + WritableUtils.writeVInt(out, 0); + } + } + + @Override + public synchronized void readFields(DataInput in) throws IOException { + int version = WritableUtils.readVInt(in); + if (version != groupFactory.version()) { + throw new IOException("Counters version mismatch, expected "+ + groupFactory.version() +" got "+ version); + } + int numFGroups = WritableUtils.readVInt(in); + fgroups.clear(); + GroupType[] groupTypes = GroupType.values(); + while (numFGroups-- > 0) { + GroupType groupType = groupTypes[WritableUtils.readVInt(in)]; + G group; + switch (groupType) { + case FILESYSTEM: // with nothing + group = groupFactory.newFileSystemGroup(); + break; + case FRAMEWORK: // with group id + group = groupFactory.newFrameworkGroup(WritableUtils.readVInt(in)); + break; + default: // Silence dumb compiler, as it would've thrown earlier + throw new IOException("Unexpected counter group type: "+ groupType); + } + group.readFields(in); + fgroups.put(group.getName(), group); + } + int numGroups = WritableUtils.readVInt(in); + while (numGroups-- > 0) { + limits.checkGroups(groups.size() + 1); + G group = groupFactory.newGenericGroup(Text.readString(in), null, limits); + group.readFields(in); + groups.put(group.getName(), group); + } + } + + /** + * Return textual representation of the counter values. + * @return the string + */ + @Override + public synchronized String toString() { + StringBuilder sb = new StringBuilder("Counters: " + countCounters()); + for (G group: this) { + sb.append("\n\t").append(group.getDisplayName()); + for (Counter counter: group) { + sb.append("\n\t\t").append(counter.getDisplayName()).append("=") + .append(counter.getValue()); + } + } + return sb.toString(); + } + + /** + * Increments multiple counters by their amounts in another Counters + * instance. + * @param other the other Counters instance + */ + public synchronized void incrAllCounters(AbstractCounters other) { + for(G right : other) { + G left = groups.get(right.getName()); + if (left == null) { + limits.checkGroups(groups.size() + 1); + left = groupFactory.newGroup(right.getName(), right.getDisplayName(), + limits); + groups.put(right.getName(), left); + } + left.incrAllCounters(right); + } + } + + @Override + @SuppressWarnings("unchecked") + public boolean equals(Object genericRight) { + if (genericRight instanceof AbstractCounters) { + return Iterators.elementsEqual(iterator(), + ((AbstractCounters)genericRight).iterator()); + } + return false; + } + + @Override + public int hashCode() { + return groups.hashCode(); + } + + /** + * Set the "writeAllCounters" option to true or false + * @param send if true all counters would be serialized, otherwise only + * framework counters would be serialized in + * {@link #write(DataOutput)} + */ + @InterfaceAudience.Private + public void setWriteAllCounters(boolean send) { + writeAllCounters = send; + } + + /** + * Get the "writeAllCounters" option + * @return true of all counters would serialized + */ + @InterfaceAudience.Private + public boolean getWriteAllCounters() { + return writeAllCounters; + } + + @InterfaceAudience.Private + public Limits limits() { + return limits; + } +} diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/CounterGroupBase.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/CounterGroupBase.java new file mode 100644 index 00000000000..b8e746d0a0b --- /dev/null +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/CounterGroupBase.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce.counters; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapreduce.Counter; + +/** + * The common counter group interface. + * + * @param type of the counter for the group + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public interface CounterGroupBase + extends Writable, Iterable { + + /** + * Get the internal name of the group + * @return the internal name + */ + String getName(); + + /** + * Get the display name of the group. + * @return the human readable name + */ + String getDisplayName(); + + /** + * Set the display name of the group + * @param displayName of the group + */ + void setDisplayName(String displayName); + + /** Add a counter to this group. + * @param counter to add + */ + void addCounter(T counter); + + /** + * Add a counter to this group + * @param name of the counter + * @param displayName of the counter + * @param value of the counter + * @return the counter + */ + T addCounter(String name, String displayName, long value); + + /** + * Find a counter in the group. + * @param counterName the name of the counter + * @param displayName the display name of the counter + * @return the counter that was found or added + */ + T findCounter(String counterName, String displayName); + + /** + * Find a counter in the group + * @param counterName the name of the counter + * @param create create the counter if not found if true + * @return the counter that was found or added or null if create is false + */ + T findCounter(String counterName, boolean create); + + /** + * Find a counter in the group. + * @param counterName the name of the counter + * @return the counter that was found or added + */ + T findCounter(String counterName); + + /** + * @return the number of counters in this group. + */ + int size(); + + /** + * Increment all counters by a group of counters + * @param rightGroup the group to be added to this group + */ + void incrAllCounters(CounterGroupBase rightGroup); +} diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/CounterGroupFactory.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/CounterGroupFactory.java new file mode 100644 index 00000000000..a402f741085 --- /dev/null +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/CounterGroupFactory.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce.counters; + +import java.util.List; +import java.util.Map; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.mapreduce.Counter; +import org.apache.hadoop.mapreduce.FileSystemCounter; +import org.apache.hadoop.mapreduce.JobCounter; +import org.apache.hadoop.mapreduce.TaskCounter; +import org.apache.hadoop.mapreduce.util.ResourceBundles; + +/** + * An abstract class to provide common implementation of the + * group factory in both mapred and mapreduce packages. + * + * @param type of the counter + * @param type of the group + */ +@InterfaceAudience.Private +public abstract class CounterGroupFactory> { + + public interface FrameworkGroupFactory { + F newGroup(String name); + } + + // Integer mapping (for serialization) for framework groups + private static final Map s2i = Maps.newHashMap(); + private static final List i2s = Lists.newArrayList(); + private static final int VERSION = 1; + private static final String FS_GROUP_NAME = FileSystemCounter.class.getName(); + + private final Map> fmap = Maps.newHashMap(); + { + // Add builtin counter class here and the version when changed. + addFrameworkGroup(TaskCounter.class); + addFrameworkGroup(JobCounter.class); + } + + // Initialize the framework counter group mapping + private synchronized > + void addFrameworkGroup(final Class cls) { + updateFrameworkGroupMapping(cls); + fmap.put(cls.getName(), newFrameworkGroupFactory(cls)); + } + + // Update static mappings (c2i, i2s) of framework groups + private static synchronized void updateFrameworkGroupMapping(Class cls) { + String name = cls.getName(); + Integer i = s2i.get(name); + if (i != null) return; + i2s.add(name); + s2i.put(name, i2s.size() - 1); + } + + /** + * Required override to return a new framework group factory + * @param type of the counter enum class + * @param cls the counter enum class + * @return a new framework group factory + */ + protected abstract > + FrameworkGroupFactory newFrameworkGroupFactory(Class cls); + + /** + * Create a new counter group + * @param name of the group + * @param limits the counters limits policy object + * @return a new counter group + */ + public G newGroup(String name, Limits limits) { + return newGroup(name, ResourceBundles.getCounterGroupName(name, name), + limits); + } + + /** + * Create a new counter group + * @param name of the group + * @param displayName of the group + * @param limits the counters limits policy object + * @return a new counter group + */ + public G newGroup(String name, String displayName, Limits limits) { + FrameworkGroupFactory gf = fmap.get(name); + if (gf != null) return gf.newGroup(name); + if (name.equals(FS_GROUP_NAME)) { + return newFileSystemGroup(); + } + return newGenericGroup(name, displayName, limits); + } + + /** + * Create a new framework group + * @param id of the group + * @return a new framework group + */ + public G newFrameworkGroup(int id) { + String name; + synchronized(CounterGroupFactory.class) { + if (id < 0 || id >= i2s.size()) throwBadFrameGroupIdException(id); + name = i2s.get(id); // should not throw here. + } + FrameworkGroupFactory gf = fmap.get(name); + if (gf == null) throwBadFrameGroupIdException(id); + return gf.newGroup(name); + } + + /** + * Get the id of a framework group + * @param name of the group + * @return the framework group id + */ + public static synchronized int getFrameworkGroupId(String name) { + Integer i = s2i.get(name); + if (i == null) throwBadFrameworkGroupNameException(name); + return i; + } + + /** + * @return the counter factory version + */ + public int version() { + return VERSION; + } + + /** + * Check whether a group name is a name of a framework group (including + * the filesystem group). + * + * @param name to check + * @return true for framework group names + */ + public static synchronized boolean isFrameworkGroup(String name) { + return s2i.get(name) != null || name.equals(FS_GROUP_NAME); + } + + private static void throwBadFrameGroupIdException(int id) { + throw new IllegalArgumentException("bad framework group id: "+ id); + } + + private static void throwBadFrameworkGroupNameException(String name) { + throw new IllegalArgumentException("bad framework group name: "+ name); + } + + /** + * Abstract factory method to create a generic (vs framework) counter group + * @param name of the group + * @param displayName of the group + * @param limits limits of the counters + * @return a new generic counter group + */ + protected abstract G newGenericGroup(String name, String displayName, + Limits limits); + + /** + * Abstract factory method to create a file system counter group + * @return a new file system counter group + */ + protected abstract G newFileSystemGroup(); +} diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java new file mode 100644 index 00000000000..7bc63a98831 --- /dev/null +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java @@ -0,0 +1,324 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce.counters; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Arrays; +import java.util.concurrent.ConcurrentMap; +import java.util.Iterator; +import java.util.Locale; +import java.util.Map; + +import com.google.common.base.Joiner; +import static com.google.common.base.Preconditions.*; +import com.google.common.collect.AbstractIterator; +import com.google.common.collect.Iterators; +import com.google.common.collect.Maps; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.io.WritableUtils; +import org.apache.hadoop.mapreduce.Counter; +import org.apache.hadoop.mapreduce.FileSystemCounter; +import org.apache.hadoop.mapreduce.util.ResourceBundles; + +/** + * An abstract class to provide common implementation of the filesystem + * counter group in both mapred and mapreduce packages. + * + * @param the type of the Counter for the group + */ +@InterfaceAudience.Private +public abstract class FileSystemCounterGroup + implements CounterGroupBase { + + static final int MAX_NUM_SCHEMES = 100; // intern/sanity check + static final ConcurrentMap schemes = Maps.newConcurrentMap(); + + // C[] would need Array.newInstance which requires a Class reference. + // Just a few local casts probably worth not having to carry it around. + private final Map map = Maps.newTreeMap(); + private String displayName; + + private static final Joiner NAME_JOINER = Joiner.on('_'); + private static final Joiner DISP_JOINER = Joiner.on(": "); + + @InterfaceAudience.Private + public class FSCounter extends AbstractCounter { + final String scheme; + final FileSystemCounter key; + private long value; + + public FSCounter(String scheme, FileSystemCounter ref) { + this.scheme = scheme; + key = ref; + } + + @Override + public String getName() { + return NAME_JOINER.join(scheme, key.name()); + } + + @Override + public String getDisplayName() { + return DISP_JOINER.join(scheme, localizeCounterName(key.name())); + } + + protected String localizeCounterName(String counterName) { + return ResourceBundles.getCounterName(FileSystemCounter.class.getName(), + counterName, counterName); + } + + @Override + public long getValue() { + return value; + } + + @Override + public void setValue(long value) { + this.value = value; + } + + @Override + public void increment(long incr) { + value += incr; + } + + @Override + public void write(DataOutput out) throws IOException { + assert false : "shouldn't be called"; + } + + @Override + public void readFields(DataInput in) throws IOException { + assert false : "shouldn't be called"; + } + } + + @Override + public String getName() { + return FileSystemCounter.class.getName(); + } + + @Override + public String getDisplayName() { + if (displayName == null) { + displayName = ResourceBundles.getCounterGroupName(getName(), + "File System Counters"); + } + return displayName; + } + + @Override + public void setDisplayName(String displayName) { + this.displayName = displayName; + } + + @Override + public void addCounter(C counter) { + C ours; + if (counter instanceof FileSystemCounterGroup.FSCounter) { + @SuppressWarnings("unchecked") + FSCounter c = (FSCounter) counter; + ours = findCounter(c.scheme, c.key); + } + else { + ours = findCounter(counter.getName()); + } + ours.setValue(counter.getValue()); + } + + @Override + public C addCounter(String name, String displayName, long value) { + C counter = findCounter(name); + counter.setValue(value); + return counter; + } + + // Parse generic counter name into [scheme, key] + private String[] parseCounterName(String counterName) { + int schemeEnd = counterName.indexOf('_'); + if (schemeEnd < 0) { + throw new IllegalArgumentException("bad fs counter name"); + } + return new String[]{counterName.substring(0, schemeEnd), + counterName.substring(schemeEnd + 1)}; + } + + @Override + public C findCounter(String counterName, String displayName) { + return findCounter(counterName); + } + + @Override + public C findCounter(String counterName, boolean create) { + try { + String[] pair = parseCounterName(counterName); + return findCounter(pair[0], FileSystemCounter.valueOf(pair[1])); + } + catch (Exception e) { + if (create) throw new IllegalArgumentException(e); + return null; + } + } + + @Override + public C findCounter(String counterName) { + return findCounter(counterName, true); + } + + @SuppressWarnings("unchecked") + public synchronized C findCounter(String scheme, FileSystemCounter key) { + final String canonicalScheme = checkScheme(scheme); + Object[] counters = map.get(canonicalScheme); + int ord = key.ordinal(); + if (counters == null) { + counters = new Object[FileSystemCounter.values().length]; + map.put(canonicalScheme, counters); + counters[ord] = newCounter(canonicalScheme, key); + } + else if (counters[ord] == null) { + counters[ord] = newCounter(canonicalScheme, key); + } + return (C) counters[ord]; + } + + private String checkScheme(String scheme) { + String fixed = scheme.toUpperCase(Locale.US); + String interned = schemes.putIfAbsent(fixed, fixed); + if (schemes.size() > MAX_NUM_SCHEMES) { + // mistakes or abuses + throw new IllegalArgumentException("too many schemes? "+ schemes.size() + + " when process scheme: "+ scheme); + } + return interned == null ? fixed : interned; + } + + /** + * Abstract factory method to create a file system counter + * @param scheme of the file system + * @param key the enum of the file system counter + * @return a new file system counter + */ + protected abstract C newCounter(String scheme, FileSystemCounter key); + + @Override + public int size() { + int n = 0; + for (Object[] counters : map.values()) { + n += numSetCounters(counters); + } + return n; + } + + @Override + @SuppressWarnings("unchecked") + public void incrAllCounters(CounterGroupBase other) { + if (checkNotNull(other, "other group") + instanceof FileSystemCounterGroup) { + for (Counter counter : other) { + FSCounter c = (FSCounter) counter; + findCounter(c.scheme, c.key) .increment(counter.getValue()); + } + } + } + + /** + * FileSystemGroup ::= #scheme (scheme #counter (key value)*)* + */ + @Override + public void write(DataOutput out) throws IOException { + WritableUtils.writeVInt(out, map.size()); // #scheme + for (Map.Entry entry : map.entrySet()) { + WritableUtils.writeString(out, entry.getKey()); // scheme + // #counter for the above scheme + WritableUtils.writeVInt(out, numSetCounters(entry.getValue())); + for (Object counter : entry.getValue()) { + if (counter == null) continue; + @SuppressWarnings("unchecked") + FSCounter c = (FSCounter) counter; + WritableUtils.writeVInt(out, c.key.ordinal()); // key + WritableUtils.writeVLong(out, c.getValue()); // value + } + } + } + + private int numSetCounters(Object[] counters) { + int n = 0; + for (Object counter : counters) if (counter != null) ++n; + return n; + } + + @Override + public void readFields(DataInput in) throws IOException { + int numSchemes = WritableUtils.readVInt(in); // #scheme + FileSystemCounter[] enums = FileSystemCounter.values(); + for (int i = 0; i < numSchemes; ++i) { + String scheme = WritableUtils.readString(in); // scheme + int numCounters = WritableUtils.readVInt(in); // #counter + for (int j = 0; j < numCounters; ++j) { + findCounter(scheme, enums[WritableUtils.readVInt(in)]) // key + .setValue(WritableUtils.readVLong(in)); // value + } + } + } + + @Override + public Iterator iterator() { + return new AbstractIterator() { + Iterator it = map.values().iterator(); + Object[] counters = it.hasNext() ? it.next() : null; + int i = 0; + @Override + protected C computeNext() { + while (counters != null) { + while (i < counters.length) { + @SuppressWarnings("unchecked") + C counter = (C) counters[i++]; + if (counter != null) return counter; + } + i = 0; + counters = it.hasNext() ? it.next() : null; + } + return endOfData(); + } + }; + } + + @Override + public synchronized boolean equals(Object genericRight) { + if (genericRight instanceof CounterGroupBase) { + @SuppressWarnings("unchecked") + CounterGroupBase right = (CounterGroupBase) genericRight; + return Iterators.elementsEqual(iterator(), right.iterator()); + } + return false; + } + + @Override + public synchronized int hashCode() { + // need to be deep as counters is an array + int hash = FileSystemCounter.class.hashCode(); + for (Object[] counters : map.values()) { + if (counters != null) hash ^= Arrays.hashCode(counters); + } + return hash; + } +} diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/FrameworkCounterGroup.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/FrameworkCounterGroup.java new file mode 100644 index 00000000000..f00982b1df7 --- /dev/null +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/FrameworkCounterGroup.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce.counters; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Arrays; +import java.util.Iterator; + +import static com.google.common.base.Preconditions.*; +import com.google.common.collect.AbstractIterator; +import com.google.common.collect.Iterators; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.io.WritableUtils; +import org.apache.hadoop.mapreduce.Counter; +import org.apache.hadoop.mapreduce.util.ResourceBundles; + +/** + * An abstract class to provide common implementation for the framework + * counter group in both mapred and mapreduce packages. + * + * @param type of the counter enum class + * @param type of the counter + */ +@InterfaceAudience.Private +public abstract class FrameworkCounterGroup, + C extends Counter> implements CounterGroupBase { + + private final Class enumClass; // for Enum.valueOf + private final Object[] counters; // local casts are OK and save a class ref + private String displayName = null; + + /** + * A counter facade for framework counters. + * Use old (which extends new) interface to make compatibility easier. + */ + @InterfaceAudience.Private + public class FrameworkCounter extends AbstractCounter { + final T key; + private long value; + + public FrameworkCounter(T ref) { + key = ref; + } + + @Override + public String getName() { + return key.name(); + } + + @Override + public String getDisplayName() { + return localizeCounterName(getName()); + } + + @Override + public long getValue() { + return value; + } + + @Override + public void setValue(long value) { + this.value = value; + } + + @Override + public void increment(long incr) { + value += incr; + } + + @Override + public void write(DataOutput out) throws IOException { + assert false : "shouldn't be called"; + } + + @Override + public void readFields(DataInput in) throws IOException { + assert false : "shouldn't be called"; + } + } + + @SuppressWarnings("unchecked") + public FrameworkCounterGroup(Class enumClass) { + this.enumClass = enumClass; + T[] enums = enumClass.getEnumConstants(); + counters = new Object[enums.length]; + } + + @Override + public String getName() { + return enumClass.getName(); + } + + @Override + public String getDisplayName() { + if (displayName == null) { + displayName = ResourceBundles.getCounterGroupName(getName(), getName()); + } + return displayName; + } + + @Override + public void setDisplayName(String displayName) { + this.displayName = displayName; + } + + private String localizeCounterName(String counterName) { + return ResourceBundles.getCounterName(getName(), counterName, counterName); + } + + private T valueOf(String name) { + return Enum.valueOf(enumClass, name); + } + + @Override + public void addCounter(C counter) { + C ours = findCounter(counter.getName()); + ours.setValue(counter.getValue()); + } + + @Override + public C addCounter(String name, String displayName, long value) { + C counter = findCounter(name); + counter.setValue(value); + return counter; + } + + @Override + public C findCounter(String counterName, String displayName) { + return findCounter(counterName); + } + + @Override + public C findCounter(String counterName, boolean create) { + try { + return findCounter(valueOf(counterName)); + } + catch (Exception e) { + if (create) throw new IllegalArgumentException(e); + return null; + } + } + + @Override + public C findCounter(String counterName) { + return findCounter(valueOf(counterName)); + } + + @SuppressWarnings("unchecked") + private C findCounter(T key) { + int i = key.ordinal(); + if (counters[i] == null) { + counters[i] = newCounter(key); + } + return (C) counters[i]; + } + + /** + * Abstract factory method for new framework counter + * @param key for the enum value of a counter + * @return a new counter for the key + */ + protected abstract C newCounter(T key); + + @Override + public int size() { + int n = 0; + for (int i = 0; i < counters.length; ++i) { + if (counters[i] != null) ++n; + } + return n; + } + + @Override + @SuppressWarnings("unchecked") + public void incrAllCounters(CounterGroupBase other) { + if (checkNotNull(other, "other counter group") + instanceof FrameworkCounterGroup) { + for (Counter counter : other) { + findCounter(((FrameworkCounter) counter).key) + .increment(counter.getValue()); + } + } + } + + /** + * FrameworkGroup ::= #counter (key value)* + */ + @Override + @SuppressWarnings("unchecked") + public void write(DataOutput out) throws IOException { + WritableUtils.writeVInt(out, size()); + for (int i = 0; i < counters.length; ++i) { + Counter counter = (C) counters[i]; + if (counter != null) { + WritableUtils.writeVInt(out, i); + WritableUtils.writeVLong(out, counter.getValue()); + } + } + } + + @Override + public void readFields(DataInput in) throws IOException { + clear(); + int len = WritableUtils.readVInt(in); + T[] enums = enumClass.getEnumConstants(); + for (int i = 0; i < len; ++i) { + int ord = WritableUtils.readVInt(in); + Counter counter = newCounter(enums[ord]); + counter.setValue(WritableUtils.readVLong(in)); + counters[ord] = counter; + } + } + + private void clear() { + for (int i = 0; i < counters.length; ++i) { + counters[i] = null; + } + } + + @Override + public Iterator iterator() { + return new AbstractIterator() { + int i = 0; + @Override + protected C computeNext() { + while (i < counters.length) { + @SuppressWarnings("unchecked") + C counter = (C) counters[i++]; + if (counter != null) return counter; + } + return endOfData(); + } + }; + } + + @Override + public boolean equals(Object genericRight) { + if (genericRight instanceof CounterGroupBase) { + @SuppressWarnings("unchecked") + CounterGroupBase right = (CounterGroupBase) genericRight; + return Iterators.elementsEqual(iterator(), right.iterator()); + } + return false; + } + + @Override + public synchronized int hashCode() { + // need to be deep as counters is an array + return Arrays.deepHashCode(new Object[]{enumClass, counters, displayName}); + } +} diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/GenericCounter.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/GenericCounter.java new file mode 100644 index 00000000000..716441795b0 --- /dev/null +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/GenericCounter.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce.counters; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.WritableUtils; + +/** + * A generic counter implementation + */ +@InterfaceAudience.Private +public class GenericCounter extends AbstractCounter { + + private String name; + private String displayName; + private long value = 0; + + public GenericCounter() { + // mostly for readFields + } + + public GenericCounter(String name, String displayName) { + this.name = name; + this.displayName = displayName; + } + + public GenericCounter(String name, String displayName, long value) { + this.name = name; + this.displayName = displayName; + this.value = value; + } + + @Override @Deprecated + public synchronized void setDisplayName(String displayName) { + this.displayName = displayName; + } + + @Override + public synchronized void readFields(DataInput in) throws IOException { + name = Text.readString(in); + displayName = in.readBoolean() ? Text.readString(in) : name; + value = WritableUtils.readVLong(in); + } + + /** + * GenericCounter ::= keyName isDistinctDisplayName [displayName] value + */ + @Override + public synchronized void write(DataOutput out) throws IOException { + Text.writeString(out, name); + boolean distinctDisplayName = ! name.equals(displayName); + out.writeBoolean(distinctDisplayName); + if (distinctDisplayName) { + Text.writeString(out, displayName); + } + WritableUtils.writeVLong(out, value); + } + + @Override + public synchronized String getName() { + return name; + } + + @Override + public synchronized String getDisplayName() { + return displayName; + } + + @Override + public synchronized long getValue() { + return value; + } + + @Override + public synchronized void setValue(long value) { + this.value = value; + } + + @Override + public synchronized void increment(long incr) { + value += incr; + } +} diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/LimitExceededException.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/LimitExceededException.java new file mode 100644 index 00000000000..a72d283ae16 --- /dev/null +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/LimitExceededException.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce.counters; + +import org.apache.hadoop.classification.InterfaceAudience; + +@InterfaceAudience.Private +public class LimitExceededException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public LimitExceededException(String msg) { + super(msg); + } + + // Only allows chaining of related exceptions + public LimitExceededException(LimitExceededException cause) { + super(cause); + } +} diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/Limits.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/Limits.java new file mode 100644 index 00000000000..24740095834 --- /dev/null +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/Limits.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce.counters; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import static org.apache.hadoop.mapreduce.MRJobConfig.*; + +@InterfaceAudience.Private +public class Limits { + + static final Configuration conf = new Configuration(); + public static final int GROUP_NAME_MAX = + conf.getInt(COUNTER_GROUP_NAME_MAX_KEY, COUNTER_GROUP_NAME_MAX_DEFAULT); + public static final int COUNTER_NAME_MAX = + conf.getInt(COUNTER_NAME_MAX_KEY, COUNTER_NAME_MAX_DEFAULT); + public static final int GROUPS_MAX = + conf.getInt(COUNTER_GROUPS_MAX_KEY, COUNTER_GROUPS_MAX_DEFAULT); + public static final int COUNTERS_MAX = + conf.getInt(COUNTERS_MAX_KEY, COUNTERS_MAX_DEFAULT); + + private int totalCounters; + private LimitExceededException firstViolation; + + public static String filterName(String name, int maxLen) { + return name.length() > maxLen ? name.substring(0, maxLen - 1) : name; + } + + public String filterCounterName(String name) { + return filterName(name, COUNTER_NAME_MAX); + } + + public String filterGroupName(String name) { + return filterName(name, GROUP_NAME_MAX); + } + + public synchronized void checkCounters(int size) { + if (firstViolation != null) { + throw new LimitExceededException(firstViolation); + } + if (size > COUNTERS_MAX) { + firstViolation = new LimitExceededException("Too many counters: "+ size + + " max="+ COUNTERS_MAX); + throw firstViolation; + } + } + + public synchronized void incrCounters() { + checkCounters(totalCounters + 1); + ++totalCounters; + } + + public synchronized void checkGroups(int size) { + if (firstViolation != null) { + throw new LimitExceededException(firstViolation); + } + if (size > GROUPS_MAX) { + firstViolation = new LimitExceededException("Too many counter groups: "+ + size +" max="+ GROUPS_MAX); + } + } + + public synchronized LimitExceededException violation() { + return firstViolation; + } +} diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/package-info.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/package-info.java new file mode 100644 index 00000000000..56d57d01044 --- /dev/null +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/counters/package-info.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This package contains the implementations of different types of + * map-reduce counters. + * + * cf. MAPREDUCE-901 for rationales. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +package org.apache.hadoop.mapreduce.counters; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java new file mode 100644 index 00000000000..99ce9033aa5 --- /dev/null +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce.jobhistory; + +import java.lang.Integer; +import java.util.Iterator; + +import org.apache.avro.Schema; + +import org.apache.avro.generic.GenericArray; +import org.apache.avro.generic.GenericData; + +public class AvroArrayUtils { + + private static final Schema ARRAY_INT + = Schema.createArray(Schema.create(Schema.Type.INT)); + + static public GenericArray NULL_PROGRESS_SPLITS_ARRAY + = new GenericData.Array(0, ARRAY_INT); + + public static GenericArray + toAvro(int values[]) { + GenericData.Array result + = new GenericData.Array(values.length, ARRAY_INT); + + for (int i = 0; i < values.length; ++i) { + result.add(values[i]); + } + + return result; + } + + public static int[] fromAvro(GenericArray avro) { + int[] result = new int[(int)avro.size()]; + + int i = 0; + + for (Iterator iter = avro.iterator(); iter.hasNext(); ++i) { + result[i] = iter.next(); + } + + return result; + } +} diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java index 36a4f95fa92..d0adc7566ca 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java @@ -22,18 +22,15 @@ import java.io.Closeable; import java.io.DataInputStream; import java.io.IOException; import java.io.EOFException; -import java.io.StringBufferInputStream; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapreduce.Counter; import org.apache.hadoop.mapreduce.CounterGroup; import org.apache.hadoop.mapreduce.Counters; import org.apache.avro.Schema; -import org.apache.avro.AvroRuntimeException; import org.apache.avro.io.Decoder; import org.apache.avro.io.JsonDecoder; import org.apache.avro.io.DatumReader; @@ -171,13 +168,10 @@ public class EventReader implements Closeable { Counters result = new Counters(); for (JhCounterGroup g : counters.groups) { CounterGroup group = - new CounterGroup(g.name.toString(), g.displayName.toString()); + result.addGroup(g.name.toString(), g.displayName.toString()); for (JhCounter c : g.counts) { - group.addCounter(new Counter(c.name.toString(), - c.displayName.toString(), - c.value)); + group.addCounter(c.name.toString(), c.displayName.toString(), c.value); } - result.addGroup(group); } return result; } diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/Events.avpr b/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/Events.avpr index 3d2bc49ef60..7825e6e258b 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/Events.avpr +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/Events.avpr @@ -125,7 +125,11 @@ {"name": "finishTime", "type": "long"}, {"name": "hostname", "type": "string"}, {"name": "state", "type": "string"}, - {"name": "counters", "type": "JhCounters"} + {"name": "counters", "type": "JhCounters"}, + {"name": "clockSplits", "type": { "type": "array", "items": "int"}}, + {"name": "cpuUsages", "type": { "type": "array", "items": "int"}}, + {"name": "vMemKbytes", "type": { "type": "array", "items": "int"}}, + {"name": "physMemKbytes", "type": { "type": "array", "items": "int"}} ] }, @@ -140,7 +144,11 @@ {"name": "finishTime", "type": "long"}, {"name": "hostname", "type": "string"}, {"name": "state", "type": "string"}, - {"name": "counters", "type": "JhCounters"} + {"name": "counters", "type": "JhCounters"}, + {"name": "clockSplits", "type": { "type": "array", "items": "int"}}, + {"name": "cpuUsages", "type": { "type": "array", "items": "int"}}, + {"name": "vMemKbytes", "type": { "type": "array", "items": "int"}}, + {"name": "physMemKbytes", "type": { "type": "array", "items": "int"}} ] }, @@ -176,7 +184,11 @@ {"name": "finishTime", "type": "long"}, {"name": "hostname", "type": "string"}, {"name": "status", "type": "string"}, - {"name": "error", "type": "string"} + {"name": "error", "type": "string"}, + {"name": "clockSplits", "type": { "type": "array", "items": "int"}}, + {"name": "cpuUsages", "type": { "type": "array", "items": "int"}}, + {"name": "vMemKbytes", "type": { "type": "array", "items": "int"}}, + {"name": "physMemKbytes", "type": { "type": "array", "items": "int"}} ] }, diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java index 130622488a1..6fa35035996 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java @@ -262,6 +262,8 @@ public class JobHistoryParser { taskInfo.finishTime = event.getFinishTime(); taskInfo.error = event.getError(); taskInfo.failedDueToAttemptId = event.getFailedAttemptID(); + info.errorInfo = "Task " + taskInfo.taskId +" failed " + + taskInfo.attemptsMap.size() + " times "; } private void handleTaskStartedEvent(TaskStartedEvent event) { @@ -321,6 +323,7 @@ public class JobHistoryParser { * The class where job information is aggregated into after parsing */ public static class JobInfo { + String errorInfo = "None"; long submitTime; long finishTime; JobID jobid; @@ -406,6 +409,7 @@ public class JobHistoryParser { public long getFinishedReduces() { return finishedReduces; } /** Get the job status */ public String getJobStatus() { return jobStatus; } + public String getErrorInfo() { return errorInfo; } /** Get the counters for the job */ public Counters getTotalCounters() { return totalCounters; } /** Get the map counters for the job */ diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java index 691b9b4cef3..e0959b08c9d 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java @@ -26,6 +26,7 @@ import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.TaskID; import org.apache.hadoop.mapreduce.TaskType; +import org.apache.hadoop.mapred.ProgressSplitsBlock; import org.apache.avro.util.Utf8; @@ -48,11 +49,19 @@ public class MapAttemptFinishedEvent implements HistoryEvent { * @param hostname Name of the host where the map executed * @param state State string for the attempt * @param counters Counters for the attempt + * @param allSplits the "splits", or a pixelated graph of various + * measurable worker node state variables against progress. + * Currently there are four; wallclock time, CPU time, + * virtual memory and physical memory. + * + * If you have no splits data, code {@code null} for this + * parameter. */ - public MapAttemptFinishedEvent(TaskAttemptID id, - TaskType taskType, String taskStatus, - long mapFinishTime, long finishTime, - String hostname, String state, Counters counters) { + public MapAttemptFinishedEvent + (TaskAttemptID id, TaskType taskType, String taskStatus, + long mapFinishTime, long finishTime, String hostname, + String state, Counters counters, + int[][] allSplits) { datum.taskid = new Utf8(id.getTaskID().toString()); datum.attemptId = new Utf8(id.toString()); datum.taskType = new Utf8(taskType.name()); @@ -62,7 +71,45 @@ public class MapAttemptFinishedEvent implements HistoryEvent { datum.hostname = new Utf8(hostname); datum.state = new Utf8(state); datum.counters = EventWriter.toAvro(counters); + + datum.clockSplits + = AvroArrayUtils.toAvro + (ProgressSplitsBlock.arrayGetWallclockTime(allSplits)); + datum.cpuUsages + = AvroArrayUtils.toAvro + (ProgressSplitsBlock.arrayGetCPUTime(allSplits)); + datum.vMemKbytes + = AvroArrayUtils.toAvro + (ProgressSplitsBlock.arrayGetVMemKbytes(allSplits)); + datum.physMemKbytes + = AvroArrayUtils.toAvro + (ProgressSplitsBlock.arrayGetPhysMemKbytes(allSplits)); } + + /** + * @deprecated please use the constructor with an additional + * argument, an array of splits arrays instead. See + * {@link org.apache.hadoop.mapred.ProgressSplitsBlock} + * for an explanation of the meaning of that parameter. + * + * Create an event for successful completion of map attempts + * @param id Task Attempt ID + * @param taskType Type of the task + * @param taskStatus Status of the task + * @param mapFinishTime Finish time of the map phase + * @param finishTime Finish time of the attempt + * @param hostname Name of the host where the map executed + * @param state State string for the attempt + * @param counters Counters for the attempt + */ + @Deprecated + public MapAttemptFinishedEvent + (TaskAttemptID id, TaskType taskType, String taskStatus, + long mapFinishTime, long finishTime, String hostname, + String state, Counters counters) { + this(id, taskType, taskStatus, mapFinishTime, finishTime, hostname, state, counters, null); + } + MapAttemptFinishedEvent() {} @@ -97,5 +144,18 @@ public class MapAttemptFinishedEvent implements HistoryEvent { public EventType getEventType() { return EventType.MAP_ATTEMPT_FINISHED; } + + public int[] getClockSplits() { + return AvroArrayUtils.fromAvro(datum.clockSplits); + } + public int[] getCpuUsages() { + return AvroArrayUtils.fromAvro(datum.cpuUsages); + } + public int[] getVMemKbytes() { + return AvroArrayUtils.fromAvro(datum.vMemKbytes); + } + public int[] getPhysMemKbytes() { + return AvroArrayUtils.fromAvro(datum.physMemKbytes); + } } diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java index e1df0a064af..fb20a2edc37 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java @@ -27,6 +27,8 @@ import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.TaskID; import org.apache.hadoop.mapreduce.TaskType; +import org.apache.hadoop.mapred.ProgressSplitsBlock; + import org.apache.avro.util.Utf8; /** @@ -50,12 +52,16 @@ public class ReduceAttemptFinishedEvent implements HistoryEvent { * @param hostname Name of the host where the attempt executed * @param state State of the attempt * @param counters Counters for the attempt + * @param allSplits the "splits", or a pixelated graph of various + * measurable worker node state variables against progress. + * Currently there are four; wallclock time, CPU time, + * virtual memory and physical memory. */ - public ReduceAttemptFinishedEvent(TaskAttemptID id, - TaskType taskType, String taskStatus, - long shuffleFinishTime, long sortFinishTime, - long finishTime, - String hostname, String state, Counters counters) { + public ReduceAttemptFinishedEvent + (TaskAttemptID id, TaskType taskType, String taskStatus, + long shuffleFinishTime, long sortFinishTime, long finishTime, + String hostname, String state, Counters counters, + int[][] allSplits) { datum.taskid = new Utf8(id.getTaskID().toString()); datum.attemptId = new Utf8(id.toString()); datum.taskType = new Utf8(taskType.name()); @@ -66,6 +72,45 @@ public class ReduceAttemptFinishedEvent implements HistoryEvent { datum.hostname = new Utf8(hostname); datum.state = new Utf8(state); datum.counters = EventWriter.toAvro(counters); + + datum.clockSplits + = AvroArrayUtils.toAvro + (ProgressSplitsBlock.arrayGetWallclockTime(allSplits)); + datum.cpuUsages + = AvroArrayUtils.toAvro + (ProgressSplitsBlock.arrayGetCPUTime(allSplits)); + datum.vMemKbytes + = AvroArrayUtils.toAvro + (ProgressSplitsBlock.arrayGetVMemKbytes(allSplits)); + datum.physMemKbytes + = AvroArrayUtils.toAvro + (ProgressSplitsBlock.arrayGetPhysMemKbytes(allSplits)); + } + + /** + * @deprecated please use the constructor with an additional + * argument, an array of splits arrays instead. See + * {@link org.apache.hadoop.mapred.ProgressSplitsBlock} + * for an explanation of the meaning of that parameter. + * + * Create an event to record completion of a reduce attempt + * @param id Attempt Id + * @param taskType Type of task + * @param taskStatus Status of the task + * @param shuffleFinishTime Finish time of the shuffle phase + * @param sortFinishTime Finish time of the sort phase + * @param finishTime Finish time of the attempt + * @param hostname Name of the host where the attempt executed + * @param state State of the attempt + * @param counters Counters for the attempt + */ + public ReduceAttemptFinishedEvent + (TaskAttemptID id, TaskType taskType, String taskStatus, + long shuffleFinishTime, long sortFinishTime, long finishTime, + String hostname, String state, Counters counters) { + this(id, taskType, taskStatus, + shuffleFinishTime, sortFinishTime, finishTime, + hostname, state, counters, null); } ReduceAttemptFinishedEvent() {} @@ -105,4 +150,17 @@ public class ReduceAttemptFinishedEvent implements HistoryEvent { } + public int[] getClockSplits() { + return AvroArrayUtils.fromAvro(datum.clockSplits); + } + public int[] getCpuUsages() { + return AvroArrayUtils.fromAvro(datum.cpuUsages); + } + public int[] getVMemKbytes() { + return AvroArrayUtils.fromAvro(datum.vMemKbytes); + } + public int[] getPhysMemKbytes() { + return AvroArrayUtils.fromAvro(datum.physMemKbytes); + } + } diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java index c23f792dc25..0a1abe882fe 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java @@ -27,6 +27,9 @@ import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.TaskID; import org.apache.hadoop.mapreduce.TaskType; +import org.apache.hadoop.mapred.ProgressSplitsBlock; +import org.apache.hadoop.mapred.TaskStatus; + import org.apache.avro.util.Utf8; /** @@ -47,11 +50,16 @@ public class TaskAttemptUnsuccessfulCompletionEvent implements HistoryEvent { * @param finishTime Finish time of the attempt * @param hostname Name of the host where the attempt executed * @param error Error string + * @param allSplits the "splits", or a pixelated graph of various + * measurable worker node state variables against progress. + * Currently there are four; wallclock time, CPU time, + * virtual memory and physical memory. */ - public TaskAttemptUnsuccessfulCompletionEvent(TaskAttemptID id, - TaskType taskType, - String status, long finishTime, - String hostname, String error) { + public TaskAttemptUnsuccessfulCompletionEvent + (TaskAttemptID id, TaskType taskType, + String status, long finishTime, + String hostname, String error, + int[][] allSplits) { datum.taskid = new Utf8(id.getTaskID().toString()); datum.taskType = new Utf8(taskType.name()); datum.attemptId = new Utf8(id.toString()); @@ -59,6 +67,40 @@ public class TaskAttemptUnsuccessfulCompletionEvent implements HistoryEvent { datum.hostname = new Utf8(hostname); datum.error = new Utf8(error); datum.status = new Utf8(status); + + datum.clockSplits + = AvroArrayUtils.toAvro + (ProgressSplitsBlock.arrayGetWallclockTime(allSplits)); + datum.cpuUsages + = AvroArrayUtils.toAvro + (ProgressSplitsBlock.arrayGetCPUTime(allSplits)); + datum.vMemKbytes + = AvroArrayUtils.toAvro + (ProgressSplitsBlock.arrayGetVMemKbytes(allSplits)); + datum.physMemKbytes + = AvroArrayUtils.toAvro + (ProgressSplitsBlock.arrayGetPhysMemKbytes(allSplits)); + } + + /** + * @deprecated please use the constructor with an additional + * argument, an array of splits arrays instead. See + * {@link org.apache.hadoop.mapred.ProgressSplitsBlock} + * for an explanation of the meaning of that parameter. + * + * Create an event to record the unsuccessful completion of attempts + * @param id Attempt ID + * @param taskType Type of the task + * @param status Status of the attempt + * @param finishTime Finish time of the attempt + * @param hostname Name of the host where the attempt executed + * @param error Error string + */ + public TaskAttemptUnsuccessfulCompletionEvent + (TaskAttemptID id, TaskType taskType, + String status, long finishTime, + String hostname, String error) { + this(id, taskType, status, finishTime, hostname, error, null); } TaskAttemptUnsuccessfulCompletionEvent() {} @@ -101,4 +143,19 @@ public class TaskAttemptUnsuccessfulCompletionEvent implements HistoryEvent { : EventType.REDUCE_ATTEMPT_KILLED); } + + + public int[] getClockSplits() { + return AvroArrayUtils.fromAvro(datum.clockSplits); + } + public int[] getCpuUsages() { + return AvroArrayUtils.fromAvro(datum.cpuUsages); + } + public int[] getVMemKbytes() { + return AvroArrayUtils.fromAvro(datum.vMemKbytes); + } + public int[] getPhysMemKbytes() { + return AvroArrayUtils.fromAvro(datum.physMemKbytes); + } + } diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java index 2566677b6a5..d5e006b776a 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java @@ -67,7 +67,7 @@ public class TaskFinishedEvent implements HistoryEvent { /** Get the task finish time */ public long getFinishTime() { return datum.finishTime; } /** Get task counters */ - Counters getCounters() { return EventReader.fromAvro(datum.counters); } + public Counters getCounters() { return EventReader.fromAvro(datum.counters); } /** Get task type */ public TaskType getTaskType() { return TaskType.valueOf(datum.taskType.toString()); diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java index 80e556bac2f..2c2b4d55606 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java @@ -114,8 +114,10 @@ public interface ClientProtocol extends VersionedProtocol { * MAPREDUCE-1664. * Version 36: Added the method getJobTrackerStatus() as part of * MAPREDUCE-2337. + * Version 37: More efficient serialization format for framework counters + * (MAPREDUCE-901) */ - public static final long versionID = 36L; + public static final long versionID = 37L; /** * Allocate a name for the job. diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/security/TokenCache.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/security/TokenCache.java index b39d90be789..6fd05feee04 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapreduce/security/TokenCache.java +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/security/TokenCache.java @@ -19,7 +19,9 @@ package org.apache.hadoop.mapreduce.security; import java.io.IOException; -import java.net.URI; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -28,16 +30,13 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.HftpFileSystem; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; -import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.JobTracker; import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier; import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.KerberosName; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; @@ -92,6 +91,13 @@ public class TokenCache { } } + static String getJTPrincipal(Configuration conf) throws IOException { + String jtHostname = JobTracker.getAddress(conf).getHostName(); + // get jobtracker principal for use as delegation token renewer + return SecurityUtil.getServerPrincipal(conf.get(JTConfig.JT_USER_NAME), + jtHostname); + } + /** * get delegation token for a specific FS * @param fs @@ -102,12 +108,11 @@ public class TokenCache { */ static void obtainTokensForNamenodesInternal(FileSystem fs, Credentials credentials, Configuration conf) throws IOException { - - // get jobtracker principal id (for the renewer) - KerberosName jtKrbName = - new KerberosName(conf.get(JTConfig.JT_USER_NAME,"")); - - String delegTokenRenewer = jtKrbName.getShortName(); + String delegTokenRenewer = getJTPrincipal(conf); + if (delegTokenRenewer == null || delegTokenRenewer.length() == 0) { + throw new IOException( + "Can't get JobTracker Kerberos principal for use as renewer"); + } boolean readFile = true; String fsName = fs.getCanonicalServiceName(); @@ -133,6 +138,16 @@ public class TokenCache { return; } } + List> tokens = fs.getDelegationTokens(delegTokenRenewer); + if (tokens != null) { + for (Token token : tokens) { + credentials.addToken(token.getService(), token); + LOG.info("Got dt for " + fs.getUri() + ";uri="+ fsName + + ";t.service="+token.getService()); + } + } + //Call getDelegationToken as well for now - for FS implementations + // which may not have implmented getDelegationTokens (hftp) Token token = fs.getDelegationToken(delegTokenRenewer); if (token != null) { Text fsNameText = new Text(fsName); diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/server/jobtracker/JTConfig.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/server/jobtracker/JTConfig.java index 4eb9188c658..ef05d2a1286 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapreduce/server/jobtracker/JTConfig.java +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/server/jobtracker/JTConfig.java @@ -89,6 +89,9 @@ public interface JTConfig extends MRConfig { "mapreduce.jobtracker.jobhistory.completed.location"; public static final String JT_JOBHISTORY_LOCATION = "mapreduce.jobtracker.jobhistory.location"; + // number of partial task progress reports we retain in job history + public static final String JT_JOBHISTORY_TASKPROGRESS_NUMBER_SPLITS = + "mapreduce.jobtracker.jobhistory.task.numberprogresssplits"; public static final String JT_AVG_BLACKLIST_THRESHOLD = "mapreduce.jobtracker.blacklist.average.threshold"; public static final String JT_SYSTEM_DIR = "mapreduce.jobtracker.system.dir"; diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java index c0a8207356a..83b4d650987 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java @@ -133,7 +133,7 @@ public class MergeManager { Counters.Counter reduceCombineInputCounter, Counters.Counter mergedMapOutputsCounter, ExceptionReporter exceptionReporter, - Progress mergePhase) { + Progress mergePhase, MapOutputFile mapOutputFile) { this.reduceId = reduceId; this.jobConf = jobConf; this.localDirAllocator = localDirAllocator; @@ -146,7 +146,7 @@ public class MergeManager { this.reduceCombineInputCounter = reduceCombineInputCounter; this.spilledRecordsCounter = spilledRecordsCounter; this.mergedMapOutputsCounter = mergedMapOutputsCounter; - this.mapOutputFile = new MapOutputFile(); + this.mapOutputFile = mapOutputFile; this.mapOutputFile.setConf(jobConf); this.localFS = localFS; diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/Shuffle.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/Shuffle.java index 177a24841b4..4b8b854952c 100644 --- a/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/Shuffle.java +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/Shuffle.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.LocalDirAllocator; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.mapred.Counters; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.MapOutputFile; import org.apache.hadoop.mapred.RawKeyValueIterator; import org.apache.hadoop.mapred.Reducer; import org.apache.hadoop.mapred.Reporter; @@ -75,7 +76,8 @@ public class Shuffle implements ExceptionReporter { TaskStatus status, Progress copyPhase, Progress mergePhase, - Task reduceTask) { + Task reduceTask, + MapOutputFile mapOutputFile) { this.reduceId = reduceId; this.jobConf = jobConf; this.umbilical = umbilical; @@ -95,7 +97,7 @@ public class Shuffle implements ExceptionReporter { spilledRecordsCounter, reduceCombineInputCounter, mergedMapOutputsCounter, - this, mergePhase); + this, mergePhase, mapOutputFile); } @SuppressWarnings("unchecked") diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/util/CountersStrings.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/util/CountersStrings.java new file mode 100644 index 00000000000..a0e542ac302 --- /dev/null +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/util/CountersStrings.java @@ -0,0 +1,285 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce.util; + +import java.text.ParseException; +import java.util.List; + +import com.google.common.collect.Lists; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.mapreduce.counters.AbstractCounters; +import org.apache.hadoop.mapreduce.Counter; +import org.apache.hadoop.mapreduce.counters.CounterGroupBase; +import org.apache.hadoop.util.StringUtils; + +/** + * String conversion utilities for counters. + * Candidate for deprecation since we start to use JSON in 0.21+ + */ +@InterfaceAudience.Private +public class CountersStrings { + private static final char GROUP_OPEN = '{'; + private static final char GROUP_CLOSE = '}'; + private static final char COUNTER_OPEN = '['; + private static final char COUNTER_CLOSE = ']'; + private static final char UNIT_OPEN = '('; + private static final char UNIT_CLOSE = ')'; + private static char[] charsToEscape = {GROUP_OPEN, GROUP_CLOSE, + COUNTER_OPEN, COUNTER_CLOSE, + UNIT_OPEN, UNIT_CLOSE}; + /** + * Make the pre 0.21 counter string (for e.g. old job history files) + * [(actual-name)(display-name)(value)] + * @param counter to stringify + * @return the stringified result + */ + public static String toEscapedCompactString(Counter counter) { + + // First up, obtain the strings that need escaping. This will help us + // determine the buffer length apriori. + String escapedName, escapedDispName; + long currentValue; + synchronized(counter) { + escapedName = escape(counter.getName()); + escapedDispName = escape(counter.getDisplayName()); + currentValue = counter.getValue(); + } + int length = escapedName.length() + escapedDispName.length() + 4; + + + length += 8; // For the following delimiting characters + StringBuilder builder = new StringBuilder(length); + builder.append(COUNTER_OPEN); + + // Add the counter name + builder.append(UNIT_OPEN); + builder.append(escapedName); + builder.append(UNIT_CLOSE); + + // Add the display name + builder.append(UNIT_OPEN); + builder.append(escapedDispName); + builder.append(UNIT_CLOSE); + + // Add the value + builder.append(UNIT_OPEN); + builder.append(currentValue); + builder.append(UNIT_CLOSE); + + builder.append(COUNTER_CLOSE); + + return builder.toString(); + } + + /** + * Make the 0.21 counter group string. + * format: {(actual-name)(display-name)(value)[][][]} + * where [] are compact strings for the counters within. + * @param type of the group + * @param group to stringify + * @return the stringified result + */ + public static > + String toEscapedCompactString(G group) { + List escapedStrs = Lists.newArrayList(); + int length; + String escapedName, escapedDispName; + synchronized(group) { + // First up, obtain the strings that need escaping. This will help us + // determine the buffer length apriori. + escapedName = escape(group.getName()); + escapedDispName = escape(group.getDisplayName()); + int i = 0; + length = escapedName.length() + escapedDispName.length(); + for (Counter counter : group) { + String escapedStr = toEscapedCompactString(counter); + escapedStrs.add(escapedStr); + length += escapedStr.length(); + } + } + length += 6; // for all the delimiting characters below + StringBuilder builder = new StringBuilder(length); + builder.append(GROUP_OPEN); // group start + + // Add the group name + builder.append(UNIT_OPEN); + builder.append(escapedName); + builder.append(UNIT_CLOSE); + + // Add the display name + builder.append(UNIT_OPEN); + builder.append(escapedDispName); + builder.append(UNIT_CLOSE); + + // write the value + for(String escaped : escapedStrs) { + builder.append(escaped); + } + + builder.append(GROUP_CLOSE); // group end + return builder.toString(); + } + + /** + * Make the pre 0.21 counters string + * @param type of the counter + * @param type of the counter group + * @param type of the counters object + * @param counters the object to stringify + * @return the string in the following format + * {(groupName)(group-displayName)[(counterName)(displayName)(value)]*}* + */ + public static , + T extends AbstractCounters> + String toEscapedCompactString(T counters) { + String[] groupsArray; + int length = 0; + synchronized(counters) { + groupsArray = new String[counters.countCounters()]; + int i = 0; + // First up, obtain the escaped string for each group so that we can + // determine the buffer length apriori. + for (G group : counters) { + String escapedString = toEscapedCompactString(group); + groupsArray[i++] = escapedString; + length += escapedString.length(); + } + } + + // Now construct the buffer + StringBuilder builder = new StringBuilder(length); + for (String group : groupsArray) { + builder.append(group); + } + return builder.toString(); + } + + // Escapes all the delimiters for counters i.e {,[,(,),],} + private static String escape(String string) { + return StringUtils.escapeString(string, StringUtils.ESCAPE_CHAR, + charsToEscape); + } + + // Unescapes all the delimiters for counters i.e {,[,(,),],} + private static String unescape(String string) { + return StringUtils.unEscapeString(string, StringUtils.ESCAPE_CHAR, + charsToEscape); + } + + // Extracts a block (data enclosed within delimeters) ignoring escape + // sequences. Throws ParseException if an incomplete block is found else + // returns null. + private static String getBlock(String str, char open, char close, + IntWritable index) throws ParseException { + StringBuilder split = new StringBuilder(); + int next = StringUtils.findNext(str, open, StringUtils.ESCAPE_CHAR, + index.get(), split); + split.setLength(0); // clear the buffer + if (next >= 0) { + ++next; // move over '(' + + next = StringUtils.findNext(str, close, StringUtils.ESCAPE_CHAR, + next, split); + if (next >= 0) { + ++next; // move over ')' + index.set(next); + return split.toString(); // found a block + } else { + throw new ParseException("Unexpected end of block", next); + } + } + return null; // found nothing + } + + /** + * Parse a pre 0.21 counters string into a counter object. + * @param type of the counter + * @param type of the counter group + * @param type of the counters object + * @param compactString to parse + * @param counters an empty counters object to hold the result + * @return the counters object holding the result + * @throws ParseException + */ + @SuppressWarnings("deprecation") + public static , + T extends AbstractCounters> + T parseEscapedCompactString(String compactString, T counters) + throws ParseException { + IntWritable index = new IntWritable(0); + + // Get the group to work on + String groupString = + getBlock(compactString, GROUP_OPEN, GROUP_CLOSE, index); + + while (groupString != null) { + IntWritable groupIndex = new IntWritable(0); + + // Get the actual name + String groupName = + getBlock(groupString, UNIT_OPEN, UNIT_CLOSE, groupIndex); + groupName = unescape(groupName); + + // Get the display name + String groupDisplayName = + getBlock(groupString, UNIT_OPEN, UNIT_CLOSE, groupIndex); + groupDisplayName = unescape(groupDisplayName); + + // Get the counters + G group = counters.getGroup(groupName); + group.setDisplayName(groupDisplayName); + + String counterString = + getBlock(groupString, COUNTER_OPEN, COUNTER_CLOSE, groupIndex); + + while (counterString != null) { + IntWritable counterIndex = new IntWritable(0); + + // Get the actual name + String counterName = + getBlock(counterString, UNIT_OPEN, UNIT_CLOSE, counterIndex); + counterName = unescape(counterName); + + // Get the display name + String counterDisplayName = + getBlock(counterString, UNIT_OPEN, UNIT_CLOSE, counterIndex); + counterDisplayName = unescape(counterDisplayName); + + // Get the value + long value = + Long.parseLong(getBlock(counterString, UNIT_OPEN, UNIT_CLOSE, + counterIndex)); + + // Add the counter + Counter counter = group.findCounter(counterName); + counter.setDisplayName(counterDisplayName); + counter.increment(value); + + // Get the next counter + counterString = + getBlock(groupString, COUNTER_OPEN, COUNTER_CLOSE, groupIndex); + } + + groupString = getBlock(compactString, GROUP_OPEN, GROUP_CLOSE, index); + } + return counters; + } +} diff --git a/mapreduce/src/java/org/apache/hadoop/mapreduce/util/ResourceBundles.java b/mapreduce/src/java/org/apache/hadoop/mapreduce/util/ResourceBundles.java new file mode 100644 index 00000000000..aede782ea04 --- /dev/null +++ b/mapreduce/src/java/org/apache/hadoop/mapreduce/util/ResourceBundles.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce.util; + +import java.util.ResourceBundle; +import java.util.MissingResourceException; + +/** + * Helper class to handle resource bundles in a saner way + */ +public class ResourceBundles { + + /** + * Get a resource bundle + * @param bundleName of the resource + * @return the resource bundle + * @throws MissingResourceException + */ + public static ResourceBundle getBundle(String bundleName) { + return ResourceBundle.getBundle(bundleName.replace('$', '_')); + } + + /** + * Get a resource given bundle name and key + * @param type of the resource + * @param bundleName name of the resource bundle + * @param key to lookup the resource + * @param suffix for the key to lookup + * @param defaultValue of the resource + * @return the resource or the defaultValue + * @throws ClassCastException if the resource found doesn't match T + */ + @SuppressWarnings("unchecked") + public static synchronized T getValue(String bundleName, String key, + String suffix, T defaultValue) { + T value; + try { + ResourceBundle bundle = getBundle(bundleName); + value = (T) bundle.getObject(getLookupKey(key, suffix)); + } + catch (Exception e) { + return defaultValue; + } + return value == null ? defaultValue : value; + } + + private static String getLookupKey(String key, String suffix) { + if (suffix == null || suffix.isEmpty()) return key; + return key + suffix; + } + + /** + * Get the counter group display name + * @param group the group name to lookup + * @param defaultValue of the group + * @return the group display name + */ + public static String getCounterGroupName(String group, String defaultValue) { + return getValue(group, "CounterGroupName", "", defaultValue); + } + + /** + * Get the counter display name + * @param group the counter group name for the counter + * @param counter the counter name to lookup + * @param defaultValue of the counter + * @return the counter display name + */ + public static String getCounterName(String group, String counter, + String defaultValue) { + return getValue(group, counter, ".name", defaultValue); + } +} diff --git a/mapreduce/src/test/mapred-site.xml b/mapreduce/src/test/mapred-site.xml index 6773c40dc57..4874e61be54 100644 --- a/mapreduce/src/test/mapred-site.xml +++ b/mapreduce/src/test/mapred-site.xml @@ -48,4 +48,8 @@ mapreduce.jobtracker.persist.jobstatus.active false + + mapreduce.task.local.output.class + org.apache.hadoop.mapred.MROutputFiles + diff --git a/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestCombineOutputCollector.java b/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestCombineOutputCollector.java new file mode 100644 index 00000000000..93560c157b1 --- /dev/null +++ b/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestCombineOutputCollector.java @@ -0,0 +1,145 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapred; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapred.Counters.Counter; +import org.apache.hadoop.mapred.IFile.Writer; +import org.apache.hadoop.mapred.Task.CombineOutputCollector; +import org.apache.hadoop.mapred.Task.TaskReporter; +import org.junit.Test; + +public class TestCombineOutputCollector { + private CombineOutputCollector coc; + + Counters.Counter outCounter = new Counters.Counter() { + + @Override + public void setValue(long value) { + // TODO Auto-generated method stub + + } + + @Override + public void setDisplayName(String displayName) { + // TODO Auto-generated method stub + + } + + @Override + public void increment(long incr) { + // TODO Auto-generated method stub + + } + + @Override + public long getValue() { + // TODO Auto-generated method stub + return 0; + } + + @Override + public String getName() { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getDisplayName() { + // TODO Auto-generated method stub + return null; + } + + @Override + public String makeEscapedCompactString() { + // TODO Auto-generated method stub + return null; + } + + @Override + public long getCounter() { + // TODO Auto-generated method stub + return 0; + } + + @Override + public boolean contentEquals(Counter counter) { + // TODO Auto-generated method stub + return false; + } + + @Override + public void write(DataOutput out) throws IOException { + } + + @Override + public void readFields(DataInput in) throws IOException { + } + }; + + @Test + public void testCustomCollect() throws Throwable { + //mock creation + TaskReporter mockTaskReporter = mock(TaskReporter.class); + Writer mockWriter = mock(Writer.class); + + Configuration conf = new Configuration(); + conf.set("mapred.combine.recordsBeforeProgress", "2"); + + coc = new CombineOutputCollector(outCounter, mockTaskReporter, conf); + coc.setWriter(mockWriter); + verify(mockTaskReporter, never()).progress(); + + coc.collect("dummy", 1); + verify(mockTaskReporter, never()).progress(); + + coc.collect("dummy", 2); + verify(mockTaskReporter, times(1)).progress(); + } + + @Test + public void testDefaultCollect() throws Throwable { + //mock creation + TaskReporter mockTaskReporter = mock(TaskReporter.class); + Writer mockWriter = mock(Writer.class); + + Configuration conf = new Configuration(); + + coc = new CombineOutputCollector(outCounter, mockTaskReporter, conf); + coc.setWriter(mockWriter); + verify(mockTaskReporter, never()).progress(); + + for(int i = 0; i < Task.DEFAULT_COMBINE_RECORDS_BEFORE_PROGRESS; i++) { + coc.collect("dummy", i); + } + verify(mockTaskReporter, times(1)).progress(); + for(int i = 0; i < Task.DEFAULT_COMBINE_RECORDS_BEFORE_PROGRESS; i++) { + coc.collect("dummy", i); + } + verify(mockTaskReporter, times(2)).progress(); + } +} diff --git a/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestIndexCache.java b/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestIndexCache.java index 2e81865743e..e0037722641 100644 --- a/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestIndexCache.java +++ b/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestIndexCache.java @@ -193,6 +193,60 @@ public class TestIndexCache extends TestCase { } } + public void testRemoveMap() throws Exception { + // This test case use two thread to call getIndexInformation and + // removeMap concurrently, in order to construct race condition. + // This test case may not repeatable. But on my macbook this test + // fails with probability of 100% on code before MAPREDUCE-2541, + // so it is repeatable in practice. + JobConf conf = new JobConf(); + FileSystem fs = FileSystem.getLocal(conf).getRaw(); + Path p = new Path(System.getProperty("test.build.data", "/tmp"), + "cache").makeQualified(fs); + fs.delete(p, true); + conf.setInt(TTConfig.TT_INDEX_CACHE, 10); + // Make a big file so removeMapThread almost surely runs faster than + // getInfoThread + final int partsPerMap = 100000; + final int bytesPerFile = partsPerMap * 24; + final IndexCache cache = new IndexCache(conf); + + final Path big = new Path(p, "bigIndex"); + final String user = + UserGroupInformation.getCurrentUser().getShortUserName(); + writeFile(fs, big, bytesPerFile, partsPerMap); + + // run multiple times + for (int i = 0; i < 20; ++i) { + Thread getInfoThread = new Thread() { + @Override + public void run() { + try { + cache.getIndexInformation("bigIndex", partsPerMap, big, user); + } catch (Exception e) { + // should not be here + } + } + }; + Thread removeMapThread = new Thread() { + @Override + public void run() { + cache.removeMap("bigIndex"); + } + }; + if (i%2==0) { + getInfoThread.start(); + removeMapThread.start(); + } else { + removeMapThread.start(); + getInfoThread.start(); + } + getInfoThread.join(); + removeMapThread.join(); + assertEquals(true, cache.checkTotalMemoryUsed()); + } + } + private static void checkRecord(IndexRecord rec, long fill) { assertEquals(fill, rec.startOffset); assertEquals(fill, rec.rawLength); diff --git a/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestJobInProgress.java b/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestJobInProgress.java index 9b0d64dde89..fc74528a016 100644 --- a/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestJobInProgress.java +++ b/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestJobInProgress.java @@ -324,7 +324,7 @@ public class TestJobInProgress extends TestCase { verify(jspy).getStatus(); verify(jspy).getProfile(); - verify(jspy).getJobCounters(); + verify(jspy, atLeastOnce()).getJobCounters(); verify(jspy, atLeastOnce()).getJobID(); verify(jspy).getStartTime(); verify(jspy).getFirstTaskLaunchTimes(); diff --git a/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestMapRed.java b/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestMapRed.java index b2b1af98523..afc0cdc0fa9 100644 --- a/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestMapRed.java +++ b/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestMapRed.java @@ -293,7 +293,7 @@ public class TestMapRed extends Configured implements Tool { ) throws IOException { if (first) { first = false; - MapOutputFile mapOutputFile = new MapOutputFile(); + MapOutputFile mapOutputFile = new MROutputFiles(); mapOutputFile.setConf(conf); Path input = mapOutputFile.getInputFile(0); FileSystem fs = FileSystem.get(conf); diff --git a/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRDFSSort.java b/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRDFSSort.java index d32f8c615a4..1b860437880 100644 --- a/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRDFSSort.java +++ b/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRDFSSort.java @@ -34,6 +34,7 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.lib.IdentityMapper; import org.apache.hadoop.mapred.lib.IdentityReducer; import org.apache.hadoop.mapred.lib.NullOutputFormat; +import org.apache.hadoop.mapreduce.FileSystemCounter; import org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -104,8 +105,8 @@ public class TestMiniMRDFSSort extends TestCase { org.apache.hadoop.mapreduce.Counters counters = sort.getResult().getCounters(); long mapInput = counters.findCounter(FileInputFormatCounter.BYTES_READ) .getValue(); - long hdfsRead = counters.findCounter(Task.FILESYSTEM_COUNTER_GROUP, - "HDFS_BYTES_READ").getValue(); + long hdfsRead = counters.findCounter("hdfs", FileSystemCounter.BYTES_READ) + .getValue(); // the hdfs read should be between 100% and 110% of the map input bytes assertTrue("map input = " + mapInput + ", hdfs read = " + hdfsRead, (hdfsRead < (mapInput * 1.1)) && diff --git a/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRWithDFS.java b/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRWithDFS.java index 4b90a675b56..2e16f844bc8 100644 --- a/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRWithDFS.java +++ b/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRWithDFS.java @@ -40,6 +40,7 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.FileSystemCounter; import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.MapReduceTestUtil; import org.apache.hadoop.mapreduce.TaskCounter; @@ -244,12 +245,10 @@ public class TestMiniMRWithDFS extends TestCase { result = launchWordCount(jobConf, inDir, outDir, input, 0, 1); assertEquals("is\t1\noom\t1\nowen\t1\n", result.output); Counters counters = result.job.getCounters(); - long hdfsRead = - counters.findCounter(Task.FILESYSTEM_COUNTER_GROUP, - Task.getFileSystemCounterNames("hdfs")[0]).getCounter(); - long hdfsWrite = - counters.findCounter(Task.FILESYSTEM_COUNTER_GROUP, - Task.getFileSystemCounterNames("hdfs")[1]).getCounter(); + long hdfsRead = counters.findCounter("HDFS", + FileSystemCounter.BYTES_READ).getValue(); + long hdfsWrite = counters.findCounter("HDFS", + FileSystemCounter.BYTES_WRITTEN).getValue(); long rawSplitBytesRead = counters.findCounter(TaskCounter.SPLIT_RAW_BYTES).getCounter(); assertEquals(result.output.length(), hdfsWrite); diff --git a/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestSeveral.java b/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestSeveral.java index d5e063ff6f4..2323d97c588 100644 --- a/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestSeveral.java +++ b/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestSeveral.java @@ -279,7 +279,7 @@ public class TestSeveral extends TestCase { ByteArrayOutputStream out = new ByteArrayOutputStream(); int exitCode = TestJobClient.runTool(conf, new JobClient(), new String[] { "-counter", jobId.toString(), - "org.apache.hadoop.mapred.Task$Counter", "MAP_INPUT_RECORDS" }, + "org.apache.hadoop.mapreduce.TaskCounter", "MAP_INPUT_RECORDS" }, out); assertEquals(0, exitCode); assertEquals(numReduces, Integer.parseInt(out.toString().trim())); diff --git a/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestTaskPerformanceSplits.java b/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestTaskPerformanceSplits.java new file mode 100644 index 00000000000..1260b148c18 --- /dev/null +++ b/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestTaskPerformanceSplits.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapred; + +import org.junit.Test; +import static org.junit.Assert.*; + +public class TestTaskPerformanceSplits { + @Test + public void testPeriodStatsets() { + PeriodicStatsAccumulator cumulative = new CumulativePeriodicStats(8); + PeriodicStatsAccumulator status = new StatePeriodicStats(8); + + cumulative.extend(0.0D, 0); + cumulative.extend(0.4375D, 700); // 200 per octant + cumulative.extend(0.5625D, 1100); // 0.5 = 900 + cumulative.extend(0.625D, 1300); + cumulative.extend(1.0D, 7901); + + int total = 0; + int[] results = cumulative.getValues(); + + for (int i = 0; i < 8; ++i) { + System.err.println("segment i = " + results[i]); + } + + assertEquals("Bad interpolation in cumulative segment 0", 200, results[0]); + assertEquals("Bad interpolation in cumulative segment 1", 200, results[1]); + assertEquals("Bad interpolation in cumulative segment 2", 200, results[2]); + assertEquals("Bad interpolation in cumulative segment 3", 300, results[3]); + assertEquals("Bad interpolation in cumulative segment 4", 400, results[4]); + assertEquals("Bad interpolation in cumulative segment 5", 2200, results[5]); + // these are rounded down + assertEquals("Bad interpolation in cumulative segment 6", 2200, results[6]); + assertEquals("Bad interpolation in cumulative segment 7", 2201, results[7]); + + status.extend(0.0D, 0); + status.extend(1.0D/16.0D, 300); // + 75 for bucket 0 + status.extend(3.0D/16.0D, 700); // + 200 for 0, +300 for 1 + status.extend(7.0D/16.0D, 2300); // + 450 for 1, + 1500 for 2, + 1050 for 3 + status.extend(1.0D, 1400); // +1125 for 3, +2100 for 4, +1900 for 5, + ; // +1700 for 6, +1500 for 7 + + results = status.getValues(); + + assertEquals("Bad interpolation in status segment 0", 275, results[0]); + assertEquals("Bad interpolation in status segment 1", 750, results[1]); + assertEquals("Bad interpolation in status segment 2", 1500, results[2]); + assertEquals("Bad interpolation in status segment 3", 2175, results[3]); + assertEquals("Bad interpolation in status segment 4", 2100, results[4]); + assertEquals("Bad interpolation in status segment 5", 1900, results[5]); + assertEquals("Bad interpolation in status segment 6", 1700, results[6]); + assertEquals("Bad interpolation in status segment 7", 1500, results[7]); + } +} diff --git a/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/SleepJob.java b/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/SleepJob.java index f6ce1cc0c48..92add201c39 100644 --- a/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/SleepJob.java +++ b/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/SleepJob.java @@ -97,6 +97,9 @@ public class SleepJob extends Configured implements Tool { public boolean nextKeyValue() throws IOException { + if (count == 0) { + return false; + } key = new IntWritable(); key.set(emitCount); int emit = emitPerMapTask / count; @@ -112,7 +115,7 @@ public class SleepJob extends Configured implements Tool { public IntWritable getCurrentValue() { return value; } public void close() throws IOException { } public float getProgress() throws IOException { - return records / ((float)count); + return count == 0 ? 100 : records / ((float)count); } }; } @@ -129,7 +132,7 @@ public class SleepJob extends Configured implements Tool { Configuration conf = context.getConfiguration(); this.mapSleepCount = conf.getInt(MAP_SLEEP_COUNT, mapSleepCount); - this.mapSleepDuration = + this.mapSleepDuration = mapSleepCount == 0 ? 0 : conf.getLong(MAP_SLEEP_TIME , 100) / mapSleepCount; } @@ -166,7 +169,7 @@ public class SleepJob extends Configured implements Tool { Configuration conf = context.getConfiguration(); this.reduceSleepCount = conf.getInt(REDUCE_SLEEP_COUNT, reduceSleepCount); - this.reduceSleepDuration = + this.reduceSleepDuration = reduceSleepCount == 0 ? 0 : conf.getLong(REDUCE_SLEEP_TIME , 100) / reduceSleepCount; } diff --git a/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/TestCounters.java b/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/TestCounters.java index c253638a76e..ccdf516d18c 100644 --- a/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/TestCounters.java +++ b/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/TestCounters.java @@ -17,17 +17,23 @@ */ package org.apache.hadoop.mapreduce; -import java.io.IOException; import java.util.Random; import org.junit.Test; import static org.junit.Assert.*; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.mapreduce.counters.LimitExceededException; +import org.apache.hadoop.mapreduce.counters.Limits; + /** * TestCounters checks the sanity and recoverability of {@code Counters} */ public class TestCounters { + static final Log LOG = LogFactory.getLog(TestCounters.class); + /** * Verify counter value works */ @@ -39,7 +45,8 @@ public class TestCounters { for (int i = 0; i < NUMBER_TESTS; i++) { long initValue = rand.nextInt(); long expectedValue = initValue; - Counter counter = new Counter("foo", "bar", expectedValue); + Counter counter = new Counters().findCounter("test", "foo"); + counter.setValue(initValue); assertEquals("Counter value is not initialized correctly", expectedValue, counter.getValue()); for (int j = 0; j < NUMBER_INC; j++) { @@ -56,4 +63,69 @@ public class TestCounters { } } + @Test public void testLimits() { + for (int i = 0; i < 3; ++i) { + // make sure limits apply to separate containers + testMaxCounters(new Counters()); + testMaxGroups(new Counters()); + } + } + + static final Enum FRAMEWORK_COUNTER = TaskCounter.CPU_MILLISECONDS; + static final long FRAMEWORK_COUNTER_VALUE = 8; + static final String FS_SCHEME = "HDFS"; + static final FileSystemCounter FS_COUNTER = FileSystemCounter.BYTES_READ; + static final long FS_COUNTER_VALUE = 10; + + private void testMaxCounters(final Counters counters) { + LOG.info("counters max="+ Limits.COUNTERS_MAX); + for (int i = 0; i < Limits.COUNTERS_MAX; ++i) { + counters.findCounter("test", "test"+ i); + } + setExpected(counters); + shouldThrow(LimitExceededException.class, new Runnable() { + public void run() { + counters.findCounter("test", "bad"); + } + }); + checkExpected(counters); + } + + private void testMaxGroups(final Counters counters) { + LOG.info("counter groups max="+ Limits.GROUPS_MAX); + for (int i = 0; i < Limits.GROUPS_MAX; ++i) { + // assuming COUNTERS_MAX > GROUPS_MAX + counters.findCounter("test"+ i, "test"); + } + setExpected(counters); + shouldThrow(LimitExceededException.class, new Runnable() { + public void run() { + counters.findCounter("bad", "test"); + } + }); + checkExpected(counters); + } + + private void setExpected(Counters counters) { + counters.findCounter(FRAMEWORK_COUNTER).setValue(FRAMEWORK_COUNTER_VALUE); + counters.findCounter(FS_SCHEME, FS_COUNTER).setValue(FS_COUNTER_VALUE); + } + + private void checkExpected(Counters counters) { + assertEquals(FRAMEWORK_COUNTER_VALUE, + counters.findCounter(FRAMEWORK_COUNTER).getValue()); + assertEquals(FS_COUNTER_VALUE, + counters.findCounter(FS_SCHEME, FS_COUNTER).getValue()); + } + + private void shouldThrow(Class ecls, Runnable runnable) { + try { + runnable.run(); + } catch (Exception e) { + assertSame(ecls, e.getClass()); + LOG.info("got expected: "+ e); + return; + } + assertTrue("Should've thrown "+ ecls.getSimpleName(), false); + } } diff --git a/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java b/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java index aaea9696381..75c38d8183a 100644 --- a/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java +++ b/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java @@ -17,6 +17,9 @@ */ package org.apache.hadoop.mapreduce.jobhistory; +import java.util.List; +import java.util.ArrayList; + import org.apache.hadoop.mapred.TaskStatus; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.TaskAttemptID; @@ -28,6 +31,15 @@ import junit.framework.TestCase; * Test various jobhistory events */ public class TestJobHistoryEvents extends TestCase { + static final int[][] NULL_SPLITS_ARRAY + = new int[org.apache.hadoop.tools.rumen.LoggedTaskAttempt.SplitVectorKind.values().length][]; + + static { + for (int i = 0; i < NULL_SPLITS_ARRAY.length; ++i) { + NULL_SPLITS_ARRAY[i] = new int[0]; + } + } + /** * Test {@link TaskAttemptStartedEvent} for various task types. */ @@ -73,7 +85,8 @@ public class TestJobHistoryEvents extends TestCase { String state) { for (TaskType t : types) { TaskAttemptUnsuccessfulCompletionEvent tauce = - new TaskAttemptUnsuccessfulCompletionEvent(id, t, state, 0L, "", ""); + new TaskAttemptUnsuccessfulCompletionEvent + (id, t, state, 0L, "", "", NULL_SPLITS_ARRAY); assertEquals(expected, tauce.getEventType()); } } diff --git a/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/security/TestBinaryTokenFile.java b/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/security/TestBinaryTokenFile.java index 674ce3b1036..fc53324d7c3 100644 --- a/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/security/TestBinaryTokenFile.java +++ b/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/security/TestBinaryTokenFile.java @@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.JobConf; @@ -147,7 +148,7 @@ public class TestBinaryTokenFile { dfsCluster.getFileSystem().getUri().toString(), 1, null, null, null, jConf); - dfsCluster.getNamesystem().getDelegationTokenSecretManager().startThreads(); + NameNodeAdapter.getDtSecretManager(dfsCluster.getNamesystem()).startThreads(); FileSystem fs = dfsCluster.getFileSystem(); p1 = new Path("file1"); @@ -177,7 +178,7 @@ public class TestBinaryTokenFile { jConf = mrCluster.createJobConf(); // provide namenodes names for the job to get the delegation tokens for - String nnUri = dfsCluster.getURI().toString(); + String nnUri = dfsCluster.getURI(0).toString(); jConf.set(MRJobConfig.JOB_NAMENODES, nnUri + "," + nnUri); // job tracker principla id.. jConf.set(JTConfig.JT_USER_NAME, "jt_id"); diff --git a/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCache.java b/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCache.java index 1b7fcc497cb..243a8e4ca07 100644 --- a/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCache.java +++ b/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCache.java @@ -30,7 +30,9 @@ import java.net.URI; import java.net.URISyntaxException; import java.security.NoSuchAlgorithmException; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import javax.crypto.KeyGenerator; @@ -38,13 +40,16 @@ import javax.crypto.spec.SecretKeySpec; import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.viewfs.ViewFileSystem; import org.apache.hadoop.hdfs.HftpFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager; import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.JobConf; @@ -148,7 +153,9 @@ public class TestTokenCache { @BeforeClass public static void setUp() throws Exception { + Configuration conf = new Configuration(); + conf.set("hadoop.security.auth_to_local", "RULE:[2:$1]"); dfsCluster = new MiniDFSCluster(conf, numSlaves, true, null); jConf = new JobConf(conf); mrCluster = new MiniMRCluster(0, 0, numSlaves, @@ -157,7 +164,7 @@ public class TestTokenCache { createTokenFileJson(); verifySecretKeysInJSONFile(); - dfsCluster.getNamesystem().getDelegationTokenSecretManager().startThreads(); + NameNodeAdapter.getDtSecretManager(dfsCluster.getNamesystem()).startThreads(); FileSystem fs = dfsCluster.getFileSystem(); p1 = new Path("file1"); @@ -223,10 +230,10 @@ public class TestTokenCache { jConf = mrCluster.createJobConf(); // provide namenodes names for the job to get the delegation tokens for - String nnUri = dfsCluster.getURI().toString(); + String nnUri = dfsCluster.getURI(0).toString(); jConf.set(MRJobConfig.JOB_NAMENODES, nnUri + "," + nnUri); // job tracker principla id.. - jConf.set(JTConfig.JT_USER_NAME, "jt_id"); + jConf.set(JTConfig.JT_USER_NAME, "jt_id/foo@BAR"); // using argument to pass the file name String[] args = { @@ -303,7 +310,7 @@ public class TestTokenCache { HftpFileSystem hfs = mock(HftpFileSystem.class); DelegationTokenSecretManager dtSecretManager = - dfsCluster.getNamesystem().getDelegationTokenSecretManager(); + NameNodeAdapter.getDtSecretManager(dfsCluster.getNamesystem()); String renewer = "renewer"; jConf.set(JTConfig.JT_USER_NAME,renewer); DelegationTokenIdentifier dtId = @@ -332,6 +339,14 @@ public class TestTokenCache { return t; }}).when(hfs).getDelegationToken(renewer); + //when(hfs.getDelegationTokens()).thenReturn((Token) t); + Mockito.doAnswer(new Answer>>(){ + @Override + public List> answer(InvocationOnMock invocation) + throws Throwable { + return Collections.singletonList(t); + }}).when(hfs).getDelegationTokens(renewer); + //when(hfs.getCanonicalServiceName).thenReturn(fs_addr); Mockito.doAnswer(new Answer(){ @Override @@ -360,4 +375,56 @@ public class TestTokenCache { } } + /** + * verify _HOST substitution + * @throws IOException + */ + @Test + public void testGetJTPrincipal() throws IOException { + String serviceName = "jt/"; + String hostName = "foo"; + String domainName = "@BAR"; + Configuration conf = new Configuration(); + conf.set(JTConfig.JT_IPC_ADDRESS, hostName + ":8888"); + conf.set(JTConfig.JT_USER_NAME, serviceName + SecurityUtil.HOSTNAME_PATTERN + + domainName); + assertEquals("Failed to substitute HOSTNAME_PATTERN with hostName", + serviceName + hostName + domainName, TokenCache.getJTPrincipal(conf)); + } + + @Test + public void testGetTokensForViewFS() throws IOException, URISyntaxException { + Configuration conf = new Configuration(jConf); + FileSystem dfs = dfsCluster.getFileSystem(); + String serviceName = dfs.getCanonicalServiceName(); + + Path p1 = new Path("/mount1"); + Path p2 = new Path("/mount2"); + p1 = dfs.makeQualified(p1); + p2 = dfs.makeQualified(p2); + + conf.set("fs.viewfs.mounttable.default.link./dir1", p1.toString()); + conf.set("fs.viewfs.mounttable.default.link./dir2", p2.toString()); + Credentials credentials = new Credentials(); + Path lp1 = new Path("viewfs:///dir1"); + Path lp2 = new Path("viewfs:///dir2"); + Path[] paths = new Path[2]; + paths[0] = lp1; + paths[1] = lp2; + TokenCache.obtainTokensForNamenodesInternal(credentials, paths, conf); + + Collection> tns = + credentials.getAllTokens(); + assertEquals("number of tokens is not 1", 1, tns.size()); + + boolean found = false; + for (Token tt : tns) { + System.out.println("token=" + tt); + if (tt.getKind().equals(DelegationTokenIdentifier.HDFS_DELEGATION_KIND) + && tt.getService().equals(new Text(serviceName))) { + found = true; + } + assertTrue("didn't find token for [" + lp1 + ", " + lp2 + "]", found); + } + } } diff --git a/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCacheOldApi.java b/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCacheOldApi.java index ba314860765..521316fa18e 100644 --- a/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCacheOldApi.java +++ b/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCacheOldApi.java @@ -40,6 +40,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; @@ -185,8 +186,7 @@ public class TestTokenCacheOldApi { createTokenFileJson(); verifySecretKeysInJSONFile(); - dfsCluster.getNamesystem() - .getDelegationTokenSecretManager().startThreads(); + NameNodeAdapter.getDtSecretManager(dfsCluster.getNamesystem()).startThreads(); FileSystem fs = dfsCluster.getFileSystem(); p1 = new Path("file1"); diff --git a/mapreduce/src/test/mapred/org/apache/hadoop/security/TestMapredGroupMappingServiceRefresh.java b/mapreduce/src/test/mapred/org/apache/hadoop/security/TestMapredGroupMappingServiceRefresh.java index b27bb487045..8b45220332b 100644 --- a/mapreduce/src/test/mapred/org/apache/hadoop/security/TestMapredGroupMappingServiceRefresh.java +++ b/mapreduce/src/test/mapred/org/apache/hadoop/security/TestMapredGroupMappingServiceRefresh.java @@ -108,7 +108,7 @@ public class TestMapredGroupMappingServiceRefresh { cluster = new MiniDFSCluster(0, config, 1, true, true, true, null, null, null, null); cluster.waitActive(); - URI uri = cluster.getURI(); + URI uri = cluster.getURI(0); MiniMRCluster miniMRCluster = new MiniMRCluster(0, uri.toString() , 3, null, null, config); diff --git a/mapreduce/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java b/mapreduce/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java index c57875a62b7..32bce5c3ce8 100644 --- a/mapreduce/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java +++ b/mapreduce/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java @@ -852,6 +852,30 @@ public class TestRumenJobTraces { public void testTopologyBuilder() throws Exception { final TopologyBuilder subject = new TopologyBuilder(); + // This 4 comes from + // TaskInProgress.ProgressibleSplitsBlock.burst().size , which + // is invisible here. + + int[][] splits = new int[4][]; + + splits[0] = new int[12]; + splits[1] = new int[12]; + splits[2] = new int[12]; + splits[3] = new int[12]; + + for (int j = 0; j < 4; ++j) { + for (int i = 0; i < 12; ++i) { + splits[j][i] = -1; + } + } + + for (int i = 0; i < 6; ++i) { + splits[0][i] = 500000 * i; + splits[1][i] = 300000 * i; + splits[2][i] = 500000; + splits[3][i] = 700000; + } + // currently we extract no host names from the Properties subject.process(new Properties()); @@ -860,16 +884,16 @@ public class TestRumenJobTraces { .valueOf("MAP"), "STATUS", 1234567890L, "/194\\.6\\.134\\.64/cluster50261\\.secondleveldomain\\.com", "SUCCESS", null)); - subject.process(new TaskAttemptUnsuccessfulCompletionEvent(TaskAttemptID - .forName("attempt_200904211745_0003_m_000004_1"), TaskType - .valueOf("MAP"), "STATUS", 1234567890L, - "/194\\.6\\.134\\.80/cluster50262\\.secondleveldomain\\.com", - "MACHINE_EXPLODED")); - subject.process(new TaskAttemptUnsuccessfulCompletionEvent(TaskAttemptID - .forName("attempt_200904211745_0003_m_000004_2"), TaskType - .valueOf("MAP"), "STATUS", 1234567890L, - "/194\\.6\\.134\\.80/cluster50263\\.secondleveldomain\\.com", - "MACHINE_EXPLODED")); + subject.process(new TaskAttemptUnsuccessfulCompletionEvent + (TaskAttemptID.forName("attempt_200904211745_0003_m_000004_1"), + TaskType.valueOf("MAP"), "STATUS", 1234567890L, + "/194\\.6\\.134\\.80/cluster50262\\.secondleveldomain\\.com", + "MACHINE_EXPLODED", splits)); + subject.process(new TaskAttemptUnsuccessfulCompletionEvent + (TaskAttemptID.forName("attempt_200904211745_0003_m_000004_2"), + TaskType.valueOf("MAP"), "STATUS", 1234567890L, + "/194\\.6\\.134\\.80/cluster50263\\.secondleveldomain\\.com", + "MACHINE_EXPLODED", splits)); subject.process(new TaskStartedEvent(TaskID .forName("task_200904211745_0003_m_000004"), 1234567890L, TaskType .valueOf("MAP"), diff --git a/mapreduce/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java b/mapreduce/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java index 9659e1c9c5c..9aa9efb8022 100644 --- a/mapreduce/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java +++ b/mapreduce/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java @@ -476,6 +476,11 @@ public class JobBuilder { } attempt.setFinishTime(event.getFinishTime()); + + attempt.arraySetClockSplits(event.getClockSplits()); + attempt.arraySetCpuUsages(event.getCpuUsages()); + attempt.arraySetVMemKbytes(event.getVMemKbytes()); + attempt.arraySetPhysMemKbytes(event.getPhysMemKbytes()); } private void processTaskAttemptStartedEvent(TaskAttemptStartedEvent event) { @@ -521,6 +526,10 @@ public class JobBuilder { attempt.setSortFinished(event.getSortFinishTime()); attempt .incorporateCounters(((ReduceAttemptFinished) event.getDatum()).counters); + attempt.arraySetClockSplits(event.getClockSplits()); + attempt.arraySetCpuUsages(event.getCpuUsages()); + attempt.arraySetVMemKbytes(event.getVMemKbytes()); + attempt.arraySetPhysMemKbytes(event.getPhysMemKbytes()); } private void processMapAttemptFinishedEvent(MapAttemptFinishedEvent event) { @@ -537,7 +546,11 @@ public class JobBuilder { // is redundant, but making this will add future-proofing. attempt.setFinishTime(event.getFinishTime()); attempt - .incorporateCounters(((MapAttemptFinished) event.getDatum()).counters); + .incorporateCounters(((MapAttemptFinished) event.getDatum()).counters); + attempt.arraySetClockSplits(event.getClockSplits()); + attempt.arraySetCpuUsages(event.getCpuUsages()); + attempt.arraySetVMemKbytes(event.getVMemKbytes()); + attempt.arraySetPhysMemKbytes(event.getPhysMemKbytes()); } private void processJobUnsuccessfulCompletionEvent( diff --git a/mapreduce/src/tools/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java b/mapreduce/src/tools/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java index 0df30e69fd8..18f518d990d 100644 --- a/mapreduce/src/tools/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java +++ b/mapreduce/src/tools/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java @@ -18,6 +18,8 @@ package org.apache.hadoop.tools.rumen; +import java.util.ArrayList; +import java.util.List; import java.util.Set; import java.util.TreeSet; @@ -71,10 +73,118 @@ public class LoggedTaskAttempt implements DeepCompare { // Initialize to default object for backward compatibility ResourceUsageMetrics metrics = new ResourceUsageMetrics(); + List clockSplits = new ArrayList(); + List cpuUsages = new ArrayList(); + List vMemKbytes = new ArrayList(); + List physMemKbytes = new ArrayList(); + LoggedTaskAttempt() { super(); } + // carries the kinds of splits vectors a LoggedTaskAttempt holds. + // + // Each enumeral has the following methods: + // get(LoggedTaskAttempt attempt) + // returns a List with the corresponding value field + // set(LoggedTaskAttempt attempt, List newValue) + // sets the value + // There is also a pair of methods get(List>) and + // set(List>, List) which correspondingly + // delivers or sets the appropriate element of the + // List> . + // This makes it easier to add another kind in the future. + public enum SplitVectorKind { + + WALLCLOCK_TIME { + @Override + public List get(LoggedTaskAttempt attempt) { + return attempt.getClockSplits(); + } + @Override + public void set(LoggedTaskAttempt attempt, List newValue) { + attempt.setClockSplits(newValue); + } + }, + + CPU_USAGE { + @Override + public List get(LoggedTaskAttempt attempt) { + return attempt.getCpuUsages(); + } + @Override + public void set(LoggedTaskAttempt attempt, List newValue) { + attempt.setCpuUsages(newValue); + } + }, + + VIRTUAL_MEMORY_KBYTES { + @Override + public List get(LoggedTaskAttempt attempt) { + return attempt.getVMemKbytes(); + } + @Override + public void set(LoggedTaskAttempt attempt, List newValue) { + attempt.setVMemKbytes(newValue); + } + }, + + PHYSICAL_MEMORY_KBYTES { + @Override + public List get(LoggedTaskAttempt attempt) { + return attempt.getPhysMemKbytes(); + } + @Override + public void set(LoggedTaskAttempt attempt, List newValue) { + attempt.setPhysMemKbytes(newValue); + } + }; + + static private final List> NULL_SPLITS_VECTOR + = new ArrayList>(); + + static { + for (SplitVectorKind kind : SplitVectorKind.values() ) { + NULL_SPLITS_VECTOR.add(new ArrayList()); + } + } + + abstract public List get(LoggedTaskAttempt attempt); + + abstract public void set(LoggedTaskAttempt attempt, List newValue); + + public List get(List> listSplits) { + return listSplits.get(this.ordinal()); + } + + public void set(List> listSplits, List newValue) { + listSplits.set(this.ordinal(), newValue); + } + + static public List> getNullSplitsVector() { + return NULL_SPLITS_VECTOR; + } + } + + /** + * + * @returns a list of all splits vectors, ordered in enumeral order + * within {@link SplitVectorKind} . Do NOT use hard-coded + * indices within the return for this with a hard-coded + * index to get individual values; use + * {@code SplitVectorKind.get(LoggedTaskAttempt)} instead. + */ + public List> allSplitVectors() { + List> result + = new ArrayList>(SplitVectorKind.values().length); + + for (SplitVectorKind kind : SplitVectorKind.values() ) { + result.add(kind.get(this)); + } + + return result; + } + static private Set alreadySeenAnySetterAttributes = new TreeSet(); @@ -89,6 +199,78 @@ public class LoggedTaskAttempt implements DeepCompare { } } + public List getClockSplits() { + return clockSplits; + } + + void setClockSplits(List clockSplits) { + this.clockSplits = clockSplits; + } + + void arraySetClockSplits(int[] clockSplits) { + List result = new ArrayList(); + + for (int i = 0; i < clockSplits.length; ++i) { + result.add(clockSplits[i]); + } + + this.clockSplits = result; + } + + public List getCpuUsages() { + return cpuUsages; + } + + void setCpuUsages(List cpuUsages) { + this.cpuUsages = cpuUsages; + } + + void arraySetCpuUsages(int[] cpuUsages) { + List result = new ArrayList(); + + for (int i = 0; i < cpuUsages.length; ++i) { + result.add(cpuUsages[i]); + } + + this.cpuUsages = result; + } + + public List getVMemKbytes() { + return vMemKbytes; + } + + void setVMemKbytes(List vMemKbytes) { + this.vMemKbytes = vMemKbytes; + } + + void arraySetVMemKbytes(int[] vMemKbytes) { + List result = new ArrayList(); + + for (int i = 0; i < vMemKbytes.length; ++i) { + result.add(vMemKbytes[i]); + } + + this.vMemKbytes = result; + } + + public List getPhysMemKbytes() { + return physMemKbytes; + } + + void setPhysMemKbytes(List physMemKbytes) { + this.physMemKbytes = physMemKbytes; + } + + void arraySetPhysMemKbytes(int[] physMemKbytes) { + List result = new ArrayList(); + + for (int i = 0; i < physMemKbytes.length; ++i) { + result.add(physMemKbytes[i]); + } + + this.physMemKbytes = result; + } + void adjustTimes(long adjustment) { startTime += adjustment; finishTime += adjustment; @@ -480,6 +662,26 @@ public class LoggedTaskAttempt implements DeepCompare { c1.deepCompare(c2, recurse); } + private void compare1(List c1, List c2, TreePath loc, + String eltname) + throws DeepInequalityException { + if (c1 == null && c2 == null) { + return; + } + + if (c1 == null || c2 == null || c1.size() != c2.size()) { + throw new DeepInequalityException + (eltname + " miscompared", new TreePath(loc, eltname)); + } + + for (int i = 0; i < c1.size(); ++i) { + if (!c1.get(i).equals(c2.get(i))) { + throw new DeepInequalityException("" + c1.get(i) + " != " + c2.get(i), + new TreePath(loc, eltname, i)); + } + } + } + public void deepCompare(DeepCompare comparand, TreePath loc) throws DeepInequalityException { if (!(comparand instanceof LoggedTaskAttempt)) { @@ -518,5 +720,10 @@ public class LoggedTaskAttempt implements DeepCompare { compare1(sortFinished, other.sortFinished, loc, "sortFinished"); compare1(location, other.location, loc, "location"); + + compare1(clockSplits, other.clockSplits, loc, "clockSplits"); + compare1(cpuUsages, other.cpuUsages, loc, "cpuUsages"); + compare1(vMemKbytes, other.vMemKbytes, loc, "vMemKbytes"); + compare1(physMemKbytes, other.physMemKbytes, loc, "physMemKbytes"); } } diff --git a/mapreduce/src/tools/org/apache/hadoop/tools/rumen/MapAttempt20LineHistoryEventEmitter.java b/mapreduce/src/tools/org/apache/hadoop/tools/rumen/MapAttempt20LineHistoryEventEmitter.java index 9b7f57dc3a0..55f9977cd48 100644 --- a/mapreduce/src/tools/org/apache/hadoop/tools/rumen/MapAttempt20LineHistoryEventEmitter.java +++ b/mapreduce/src/tools/org/apache/hadoop/tools/rumen/MapAttempt20LineHistoryEventEmitter.java @@ -68,10 +68,13 @@ public class MapAttempt20LineHistoryEventEmitter extends (MapAttempt20LineHistoryEventEmitter) thatg; if (finishTime != null && "success".equalsIgnoreCase(status)) { - return new MapAttemptFinishedEvent(taskAttemptID, - that.originalTaskType, status, Long.parseLong(finishTime), Long - .parseLong(finishTime), hostName, state, - maybeParseCounters(counters)); + return new MapAttemptFinishedEvent + (taskAttemptID, + that.originalTaskType, status, + Long.parseLong(finishTime), + Long.parseLong(finishTime), + hostName, state, maybeParseCounters(counters), + null); } } @@ -88,5 +91,4 @@ public class MapAttempt20LineHistoryEventEmitter extends List nonFinalSEEs() { return nonFinals; } - } diff --git a/mapreduce/src/tools/org/apache/hadoop/tools/rumen/MapTaskAttemptInfo.java b/mapreduce/src/tools/org/apache/hadoop/tools/rumen/MapTaskAttemptInfo.java index ebb6f62cf24..2406b9daae5 100644 --- a/mapreduce/src/tools/org/apache/hadoop/tools/rumen/MapTaskAttemptInfo.java +++ b/mapreduce/src/tools/org/apache/hadoop/tools/rumen/MapTaskAttemptInfo.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.tools.rumen; +import java.util.List; + import org.apache.hadoop.mapred.TaskStatus.State; /** @@ -26,11 +28,33 @@ import org.apache.hadoop.mapred.TaskStatus.State; public class MapTaskAttemptInfo extends TaskAttemptInfo { private long runtime; - public MapTaskAttemptInfo(State state, TaskInfo taskInfo, long runtime) { - super(state, taskInfo); + public MapTaskAttemptInfo(State state, TaskInfo taskInfo, + long runtime, List> allSplits) { + super(state, taskInfo, + allSplits == null + ? LoggedTaskAttempt.SplitVectorKind.getNullSplitsVector() + : allSplits); this.runtime = runtime; } + /** + * + * @deprecated please use the constructor with + * {@code (state, taskInfo, runtime, + * List> allSplits)} + * instead. + * + * see {@link LoggedTaskAttempt} for an explanation of + * {@code allSplits}. + * + * If there are no known splits, use {@code null}. + */ + @Deprecated + public MapTaskAttemptInfo(State state, TaskInfo taskInfo, + long runtime) { + this(state, taskInfo, runtime, null); + } + @Override public long getRuntime() { return getMapRuntime(); diff --git a/mapreduce/src/tools/org/apache/hadoop/tools/rumen/ReduceAttempt20LineHistoryEventEmitter.java b/mapreduce/src/tools/org/apache/hadoop/tools/rumen/ReduceAttempt20LineHistoryEventEmitter.java index 974475aa548..234a4338406 100644 --- a/mapreduce/src/tools/org/apache/hadoop/tools/rumen/ReduceAttempt20LineHistoryEventEmitter.java +++ b/mapreduce/src/tools/org/apache/hadoop/tools/rumen/ReduceAttempt20LineHistoryEventEmitter.java @@ -28,8 +28,8 @@ import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.jobhistory.HistoryEvent; import org.apache.hadoop.mapreduce.jobhistory.ReduceAttemptFinishedEvent; -public class ReduceAttempt20LineHistoryEventEmitter extends - TaskAttempt20LineEventEmitter { +public class ReduceAttempt20LineHistoryEventEmitter + extends TaskAttempt20LineEventEmitter { static List nonFinals = new LinkedList(); @@ -71,10 +71,15 @@ public class ReduceAttempt20LineHistoryEventEmitter extends ReduceAttempt20LineHistoryEventEmitter that = (ReduceAttempt20LineHistoryEventEmitter) thatg; - return new ReduceAttemptFinishedEvent(taskAttemptID, - that.originalTaskType, status, Long.parseLong(shuffleFinish), - Long.parseLong(sortFinish), Long.parseLong(finishTime), hostName, - state, maybeParseCounters(counters)); + return new ReduceAttemptFinishedEvent + (taskAttemptID, + that.originalTaskType, status, + Long.parseLong(shuffleFinish), + Long.parseLong(sortFinish), + Long.parseLong(finishTime), + hostName, + state, maybeParseCounters(counters), + null); } } diff --git a/mapreduce/src/tools/org/apache/hadoop/tools/rumen/ReduceTaskAttemptInfo.java b/mapreduce/src/tools/org/apache/hadoop/tools/rumen/ReduceTaskAttemptInfo.java index 96698597dde..bd9fe9aca40 100644 --- a/mapreduce/src/tools/org/apache/hadoop/tools/rumen/ReduceTaskAttemptInfo.java +++ b/mapreduce/src/tools/org/apache/hadoop/tools/rumen/ReduceTaskAttemptInfo.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.tools.rumen; +import java.util.List; + import org.apache.hadoop.mapred.TaskStatus.State; /** @@ -29,13 +31,35 @@ public class ReduceTaskAttemptInfo extends TaskAttemptInfo { private long reduceTime; public ReduceTaskAttemptInfo(State state, TaskInfo taskInfo, long shuffleTime, - long mergeTime, long reduceTime) { - super(state, taskInfo); + long mergeTime, long reduceTime, List> allSplits) { + super(state, taskInfo, + allSplits == null + ? LoggedTaskAttempt.SplitVectorKind.getNullSplitsVector() + : allSplits); this.shuffleTime = shuffleTime; this.mergeTime = mergeTime; this.reduceTime = reduceTime; } + + /** + * + * @deprecated please use the constructor with + * {@code (state, taskInfo, shuffleTime, mergeTime, reduceTime + * List> allSplits)} + * instead. + * + * see {@link LoggedTaskAttempt} for an explanation of + * {@code allSplits}. + * + * If there are no known splits, use {@code null}. + */ + @Deprecated + public ReduceTaskAttemptInfo(State state, TaskInfo taskInfo, long shuffleTime, + long mergeTime, long reduceTime) { + this(state, taskInfo, shuffleTime, mergeTime, reduceTime, null); + } + /** * Get the runtime for the reduce phase of the reduce task-attempt. * @@ -67,5 +91,4 @@ public class ReduceTaskAttemptInfo extends TaskAttemptInfo { public long getRuntime() { return (getShuffleRuntime() + getMergeRuntime() + getReduceRuntime()); } - } diff --git a/mapreduce/src/tools/org/apache/hadoop/tools/rumen/TaskAttempt20LineEventEmitter.java b/mapreduce/src/tools/org/apache/hadoop/tools/rumen/TaskAttempt20LineEventEmitter.java index 7814c480848..77f35a7ceb5 100644 --- a/mapreduce/src/tools/org/apache/hadoop/tools/rumen/TaskAttempt20LineEventEmitter.java +++ b/mapreduce/src/tools/org/apache/hadoop/tools/rumen/TaskAttempt20LineEventEmitter.java @@ -138,9 +138,10 @@ public abstract class TaskAttempt20LineEventEmitter extends HistoryEventEmitter TaskAttempt20LineEventEmitter that = (TaskAttempt20LineEventEmitter) thatg; - return new TaskAttemptUnsuccessfulCompletionEvent(taskAttemptID, - that.originalTaskType, status, Long.parseLong(finishTime), - hostName, error); + return new TaskAttemptUnsuccessfulCompletionEvent + (taskAttemptID, + that.originalTaskType, status, Long.parseLong(finishTime), + hostName, error, null); } return null; diff --git a/mapreduce/src/tools/org/apache/hadoop/tools/rumen/TaskAttemptInfo.java b/mapreduce/src/tools/org/apache/hadoop/tools/rumen/TaskAttemptInfo.java index b8921e92f28..d13974f2ca2 100644 --- a/mapreduce/src/tools/org/apache/hadoop/tools/rumen/TaskAttemptInfo.java +++ b/mapreduce/src/tools/org/apache/hadoop/tools/rumen/TaskAttemptInfo.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.tools.rumen; +import java.util.List; + import org.apache.hadoop.mapred.TaskStatus.State; /** @@ -27,13 +29,22 @@ public abstract class TaskAttemptInfo { protected final State state; protected final TaskInfo taskInfo; - protected TaskAttemptInfo(State state, TaskInfo taskInfo) { + protected final List> allSplits; + + protected TaskAttemptInfo + (State state, TaskInfo taskInfo, List> allSplits) { if (state == State.SUCCEEDED || state == State.FAILED) { this.state = state; } else { throw new IllegalArgumentException("status cannot be " + state); } this.taskInfo = taskInfo; + this.allSplits = allSplits; + } + + protected TaskAttemptInfo + (State state, TaskInfo taskInfo) { + this(state, taskInfo, LoggedTaskAttempt.SplitVectorKind.getNullSplitsVector()); } /** @@ -60,4 +71,8 @@ public abstract class TaskAttemptInfo { public TaskInfo getTaskInfo() { return taskInfo; } + + public List getSplitVector(LoggedTaskAttempt.SplitVectorKind kind) { + return kind.get(allSplits); + } } diff --git a/mapreduce/src/tools/org/apache/hadoop/tools/rumen/ZombieJob.java b/mapreduce/src/tools/org/apache/hadoop/tools/rumen/ZombieJob.java index 838995d6c8f..45364131865 100644 --- a/mapreduce/src/tools/org/apache/hadoop/tools/rumen/ZombieJob.java +++ b/mapreduce/src/tools/org/apache/hadoop/tools/rumen/ZombieJob.java @@ -537,7 +537,8 @@ public class ZombieJob implements JobStory { } taskTime = sanitizeTaskRuntime(taskTime, loggedAttempt.getAttemptID()); taskTime *= scaleFactor; - return new MapTaskAttemptInfo(state, taskInfo, taskTime); + return new MapTaskAttemptInfo + (state, taskInfo, taskTime, loggedAttempt.allSplitVectors()); } else { throw new IllegalArgumentException("taskType can only be MAP: " + loggedTask.getTaskType()); @@ -584,6 +585,9 @@ public class ZombieJob implements JobStory { private TaskAttemptInfo getTaskAttemptInfo(LoggedTask loggedTask, LoggedTaskAttempt loggedAttempt) { TaskInfo taskInfo = getTaskInfo(loggedTask); + + List> allSplitVectors = loggedAttempt.allSplitVectors(); + State state = convertState(loggedAttempt.getResult()); if (loggedTask.getTaskType() == Values.MAP) { long taskTime; @@ -594,7 +598,7 @@ public class ZombieJob implements JobStory { taskTime = loggedAttempt.getFinishTime() - loggedAttempt.getStartTime(); } taskTime = sanitizeTaskRuntime(taskTime, loggedAttempt.getAttemptID()); - return new MapTaskAttemptInfo(state, taskInfo, taskTime); + return new MapTaskAttemptInfo(state, taskInfo, taskTime, allSplitVectors); } else if (loggedTask.getTaskType() == Values.REDUCE) { long startTime = loggedAttempt.getStartTime(); long mergeDone = loggedAttempt.getSortFinished(); @@ -605,7 +609,8 @@ public class ZombieJob implements JobStory { // haven't seen reduce task with startTime=0 ever. But if this happens, // make up a reduceTime with no shuffle/merge. long reduceTime = makeUpReduceRuntime(state); - return new ReduceTaskAttemptInfo(state, taskInfo, 0, 0, reduceTime); + return new ReduceTaskAttemptInfo + (state, taskInfo, 0, 0, reduceTime, allSplitVectors); } else { if (shuffleDone <= 0) { shuffleDone = startTime; @@ -619,7 +624,7 @@ public class ZombieJob implements JobStory { reduceTime = sanitizeTaskRuntime(reduceTime, loggedAttempt.getAttemptID()); return new ReduceTaskAttemptInfo(state, taskInfo, shuffleTime, - mergeTime, reduceTime); + mergeTime, reduceTime, allSplitVectors); } } else { throw new IllegalArgumentException("taskType for " @@ -700,7 +705,8 @@ public class ZombieJob implements JobStory { runtime = makeUpMapRuntime(state, locality); runtime = sanitizeTaskRuntime(runtime, makeTaskAttemptID(taskType, taskNumber, taskAttemptNumber).toString()); - TaskAttemptInfo tai = new MapTaskAttemptInfo(state, taskInfo, runtime); + TaskAttemptInfo tai + = new MapTaskAttemptInfo(state, taskInfo, runtime, null); return tai; } else if (taskType == TaskType.REDUCE) { State state = State.SUCCEEDED; @@ -711,8 +717,8 @@ public class ZombieJob implements JobStory { // TODO make up state // state = makeUpState(taskAttemptNumber, job.getReducerTriesToSucceed()); reduceTime = makeUpReduceRuntime(state); - TaskAttemptInfo tai = new ReduceTaskAttemptInfo(state, taskInfo, - shuffleTime, sortTime, reduceTime); + TaskAttemptInfo tai = new ReduceTaskAttemptInfo + (state, taskInfo, shuffleTime, sortTime, reduceTime, null); return tai; } diff --git a/mapreduce/src/webapps/job/jobdetailshistory.jsp b/mapreduce/src/webapps/job/jobdetailshistory.jsp index 68efc294969..2b0a014b3f5 100644 --- a/mapreduce/src/webapps/job/jobdetailshistory.jsp +++ b/mapreduce/src/webapps/job/jobdetailshistory.jsp @@ -45,6 +45,7 @@ <%! static SimpleDateFormat dateFormat = new SimpleDateFormat("d-MMM-yyyy HH:mm:ss") ; %> <% String logFile = request.getParameter("logFile"); + String reasonforFailure = " "; final Path jobFile = new Path(logFile); String jobid = JobHistory.getJobIDFromHistoryFilePath(jobFile).toString(); @@ -55,6 +56,8 @@ if (job == null) { return; } + if (job.getJobStatus().equals("FAILED")) + reasonforFailure = job.getErrorInfo(); %> @@ -78,6 +81,7 @@ Launched At: <%=StringUtils.getFormattedTimeWithDiff(dateFormat, job.getLaunchTime(), job.getSubmitTime()) %>
    Finished At: <%=StringUtils.getFormattedTimeWithDiff(dateFormat, job.getFinishTime(), job.getLaunchTime()) %>
    Status: <%= ((job.getJobStatus()) == null ? "Incomplete" :job.getJobStatus()) %>
    +ReasonForFailure: <%=reasonforFailure %>
    <% HistoryViewer.SummarizedJob sj = new HistoryViewer.SummarizedJob(job); %> diff --git a/pom.xml b/pom.xml index ba41a928c33..004706d72de 100644 --- a/pom.xml +++ b/pom.xml @@ -53,6 +53,11 @@ maven-assembly-plugin 2.2-beta-3 + + org.apache.maven.plugins + maven-deploy-plugin + 2.5 + org.apache.rat apache-rat-plugin @@ -74,9 +79,6 @@ 1.6 - - unix - @@ -103,6 +105,12 @@ + + maven-deploy-plugin + + true + + org.apache.rat apache-rat-plugin