Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'trunk' into HADOOP-12111

This commit is contained in:
Allen Wittenauer 2015-06-30 11:51:13 -07:00
parent 1b80406285 fe6c1bd73a
commit 084becdd72
129 changed files with 4332 additions and 1497 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
View File

@ -160,6 +160,12 @@ public class AuthenticationFilter implements Filter {
*/
public static final String COOKIE_PATH = "cookie.path";
/**
* Constant for the configuration property
* that indicates the persistence of the HTTP cookie.
*/
public static final String COOKIE_PERSISTENT = "cookie.persistent";
/**
* Constant for the configuration property that indicates the name of the
* SignerSecretProvider class to use.
@ -187,6 +193,7 @@ public class AuthenticationFilter implements Filter {
private long validity;
private String cookieDomain;
private String cookiePath;
private boolean isCookiePersistent;
private boolean isInitializedByTomcat;
/**
@ -228,6 +235,9 @@ public class AuthenticationFilter implements Filter {
cookieDomain = config.getProperty(COOKIE_DOMAIN, null);
cookiePath = config.getProperty(COOKIE_PATH, null);
isCookiePersistent = Boolean.parseBoolean(
config.getProperty(COOKIE_PERSISTENT, "false"));
}
protected void initializeAuthHandler(String authHandlerClassName, FilterConfig filterConfig)
@ -371,6 +381,15 @@ public class AuthenticationFilter implements Filter {
return cookiePath;
}
/**
* Returns the cookie persistence to use for the HTTP cookie.
*
* @return the cookie persistence to use for the HTTP cookie.
*/
protected boolean isCookiePersistent() {
return isCookiePersistent;
}
/**
* Destroys the filter.
* <p>
@ -549,7 +568,8 @@ public class AuthenticationFilter implements Filter {
if (newToken && !token.isExpired() && token != AuthenticationToken.ANONYMOUS) {
String signedToken = signer.sign(token.toString());
createAuthCookie(httpResponse, signedToken, getCookieDomain(),
getCookiePath(), token.getExpires(), isHttps);
getCookiePath(), token.getExpires(),
isCookiePersistent(), isHttps);
}
doFilter(filterChain, httpRequest, httpResponse);
}
@ -569,7 +589,7 @@ public class AuthenticationFilter implements Filter {
if (unauthorizedResponse) {
if (!httpResponse.isCommitted()) {
createAuthCookie(httpResponse, "", getCookieDomain(),
getCookiePath(), 0, isHttps);
getCookiePath(), 0, isCookiePersistent(), isHttps);
// If response code is 401. Then WWW-Authenticate Header should be
// present.. reset to 403 if not found..
if ((errCode == HttpServletResponse.SC_UNAUTHORIZED)
@ -614,6 +634,7 @@ public class AuthenticationFilter implements Filter {
* @param isSecure is the cookie secure?
* @param token the token.
* @param expires the cookie expiration time.
* @param isCookiePersistent whether the cookie is persistent or not.
*
* XXX the following code duplicate some logic in Jetty / Servlet API,
* because of the fact that Hadoop is stuck at servlet 2.5 and jetty 6
@ -621,6 +642,7 @@ public class AuthenticationFilter implements Filter {
*/
public static void createAuthCookie(HttpServletResponse resp, String token,
String domain, String path, long expires,
boolean isCookiePersistent,
boolean isSecure) {
StringBuilder sb = new StringBuilder(AuthenticatedURL.AUTH_COOKIE)
.append("=");
@ -636,7 +658,7 @@ public class AuthenticationFilter implements Filter {
sb.append("; Domain=").append(domain);
}
if (expires >= 0) {
if (expires >= 0 && isCookiePersistent) {
Date date = new Date(expires);
SimpleDateFormat df = new SimpleDateFormat("EEE, " +
"dd-MMM-yyyy HH:mm:ss zzz");

8
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -53,6 +53,9 @@ Trunk (Unreleased)
IMPROVEMENTS
HADOOP-11203. Allow ditscp to accept bandwitdh in fraction MegaBytes
(Raju Bairishetti via amareshwari)
HADOOP-8017. Configure hadoop-main pom to get rid of M2E plugin execution
not covered (Eric Charles via bobby)
@ -656,6 +659,8 @@ Release 2.8.0 - UNRELEASED
HADOOP-7139. Allow appending to existing SequenceFiles
(kanaka kumar avvaru via vinayakumarb)
HADOOP-11958. MetricsSystemImpl fails to show backtrace when an error
occurs (Jason Lowe via jeagles)
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
@ -676,6 +681,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11885. hadoop-dist dist-layout-stitching.sh does not work with dash.
(wang)
HADOOP-12036. Consolidate all of the cmake extensions in one directory
(alanburlison via cmccabe)
BUG FIXES
HADOOP-11802: DomainSocketWatcher thread terminates sometimes after there

207
hadoop-common-project/hadoop-common/HadoopCommon.cmake Normal file
View File

@ -0,0 +1,207 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
# Common CMake utilities and configuration, shared by all Native components.
#
#
# Platform-specific prerequisite checks.
#
if(CMAKE_SYSTEM_NAME STREQUAL "SunOS")
# Only 64-bit Java is supported.
if(NOT JVM_ARCH_DATA_MODEL EQUAL 64)
message(FATAL_ERROR "Unrecognised JVM_ARCH_DATA_MODEL '${JVM_ARCH_DATA_MODEL}'. "
"A 64-bit JVM must be used on Solaris, make sure that one is installed and, "
"if necessary, the MAVEN_OPTS environment variable includes '-d64'")
endif()
# Only gcc is suported for now.
if(NOT(CMAKE_COMPILER_IS_GNUCC AND CMAKE_COMPILER_IS_GNUCXX))
message(FATAL_ERROR "Only gcc is supported on Solaris")
endif()
endif()
#
# Helper functions and macros.
#
# Add flags to all the CMake compiler variables
macro(hadoop_add_compiler_flags FLAGS)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FLAGS}")
endmacro()
# Add flags to all the CMake linker variables
macro(hadoop_add_linker_flags FLAGS)
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${FLAGS}")
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${FLAGS}")
set(CMAKE_STATIC_LINKER_FLAGS "${CMAKE_STATIC_LINKER_FLAGS} ${FLAGS}")
endmacro()
# Compile a library with both shared and static variants.
function(hadoop_add_dual_library LIBNAME)
add_library(${LIBNAME} SHARED ${ARGN})
add_library(${LIBNAME}_static STATIC ${ARGN})
set_target_properties(${LIBNAME}_static PROPERTIES OUTPUT_NAME ${LIBNAME})
endfunction()
# Link both a static and a dynamic target against some libraries.
function(hadoop_target_link_dual_libraries LIBNAME)
target_link_libraries(${LIBNAME} ${ARGN})
target_link_libraries(${LIBNAME}_static ${ARGN})
endfunction()
# Set all the output directories to the same place.
function(hadoop_output_directory TGT DIR)
set_target_properties(${TGT} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
set_target_properties(${TGT} PROPERTIES ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
set_target_properties(${TGT} PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
endfunction()
# Set the target directories for dynamic and static builds.
function(hadoop_dual_output_directory TGT DIR)
hadoop_output_directory(${TGT} "${DIR}")
hadoop_output_directory(${TGT}_static "${DIR}")
endfunction()
# Alter the behavior of find_package and find_library so that we find only
# shared libraries with a given version suffix. You should save
# CMAKE_FIND_LIBRARY_SUFFIXES before calling this function and restore it
# afterwards. On Windows this function is a no-op. Windows does not encode
# version number information information into library path names.
macro(hadoop_set_find_shared_library_version LVERS)
if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
# Mac OS uses .dylib
set(CMAKE_FIND_LIBRARY_SUFFIXES ".${LVERS}.dylib")
elseif(${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD")
# FreeBSD has always .so installed.
set(CMAKE_FIND_LIBRARY_SUFFIXES ".so")
elseif(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
# Windows doesn't support finding shared libraries by version.
else()
# Most UNIX variants use .so
set(CMAKE_FIND_LIBRARY_SUFFIXES ".so.${LVERS}")
endif()
endmacro()
# Alter the behavior of find_package and find_library so that we find only
# shared libraries without any version suffix. You should save
# CMAKE_FIND_LIBRARY_SUFFIXES before calling this function and restore it
# afterwards. On Windows this function is a no-op. Windows does not encode
# version number information information into library path names.
macro(hadoop_set_find_shared_library_without_version)
if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
# Mac OS uses .dylib
set(CMAKE_FIND_LIBRARY_SUFFIXES ".dylib")
elseif(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
# No effect
else()
# Most UNIX variants use .so
set(CMAKE_FIND_LIBRARY_SUFFIXES ".so")
endif()
endmacro()
#
# Configuration.
#
# Initialise the shared gcc/g++ flags if they aren't already defined.
if(NOT DEFINED GCC_SHARED_FLAGS)
set(GCC_SHARED_FLAGS "-g -O2 -Wall -pthread -D_FILE_OFFSET_BITS=64")
endif()
# Add in support other compilers here, if necessary,
# the assumption is that GCC or a GCC-compatible compiler is being used.
# Set the shared GCC-compatible compiler and linker flags.
hadoop_add_compiler_flags("${GCC_SHARED_FLAGS}")
hadoop_add_linker_flags("${LINKER_SHARED_FLAGS}")
#
# Linux-specific configuration.
#
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
# Make GNU extensions available.
hadoop_add_compiler_flags("-D_GNU_SOURCE")
# If JVM_ARCH_DATA_MODEL is 32, compile all binaries as 32-bit.
if(JVM_ARCH_DATA_MODEL EQUAL 32)
# Force 32-bit code generation on amd64/x86_64, ppc64, sparc64
if(CMAKE_COMPILER_IS_GNUCC AND CMAKE_SYSTEM_PROCESSOR MATCHES ".*64")
hadoop_add_compiler_flags("-m32")
hadoop_add_linker_flags("-m32")
endif()
# Set CMAKE_SYSTEM_PROCESSOR to ensure that find_package(JNI) will use 32-bit libraries
if(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64")
set(CMAKE_SYSTEM_PROCESSOR "i686")
endif()
endif()
# Determine float ABI of JVM on ARM.
if(CMAKE_SYSTEM_PROCESSOR MATCHES "^arm")
find_program(READELF readelf)
if(READELF MATCHES "NOTFOUND")
message(WARNING "readelf not found; JVM float ABI detection disabled")
else(READELF MATCHES "NOTFOUND")
execute_process(
COMMAND ${READELF} -A ${JAVA_JVM_LIBRARY}
OUTPUT_VARIABLE JVM_ELF_ARCH
ERROR_QUIET)
if(NOT JVM_ELF_ARCH MATCHES "Tag_ABI_VFP_args: VFP registers")
# Test compilation with -mfloat-abi=softfp using an arbitrary libc function
# (typically fails with "fatal error: bits/predefs.h: No such file or directory"
# if soft-float dev libraries are not installed)
message("Soft-float JVM detected")
include(CMakePushCheckState)
cmake_push_check_state()
set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -mfloat-abi=softfp")
include(CheckSymbolExists)
check_symbol_exists(exit stdlib.h SOFTFP_AVAILABLE)
if(NOT SOFTFP_AVAILABLE)
message(FATAL_ERROR "Soft-float dev libraries required (e.g. 'apt-get install libc6-dev-armel' on Debian/Ubuntu)")
endif()
cmake_pop_check_state()
hadoop_add_compiler_flags("-mfloat-abi=softfp")
endif()
endif()
endif()
#
# Solaris-specific configuration.
#
elseif(CMAKE_SYSTEM_NAME STREQUAL "SunOS")
# Solaris flags. 64-bit compilation is mandatory, and is checked earlier.
hadoop_add_compiler_flags("-m64 -D__EXTENSIONS__ -D_POSIX_PTHREAD_SEMANTICS -D_XOPEN_SOURCE=500")
hadoop_add_linker_flags("-m64")
# CMAKE_SYSTEM_PROCESSOR is set to the output of 'uname -p', which on Solaris is
# the 'lowest' ISA supported, i.e. 'i386' or 'sparc'. However in order for the
# standard CMake modules to look in the right places it needs to reflect the required
# compilation mode, i.e. 64 bit. We therefore force it to either 'amd64' or 'sparcv9'.
if(CMAKE_SYSTEM_PROCESSOR STREQUAL "i386")
set(CMAKE_SYSTEM_PROCESSOR "amd64")
set(CMAKE_LIBRARY_ARCHITECTURE "amd64")
elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "sparc")
set(CMAKE_SYSTEM_PROCESSOR STREQUAL "sparcv9")
set(CMAKE_LIBRARY_ARCHITECTURE "sparcv9")
else()
message(FATAL_ERROR "Unrecognised CMAKE_SYSTEM_PROCESSOR ${CMAKE_SYSTEM_PROCESSOR}")
endif()
endif()

97
hadoop-common-project/hadoop-common/HadoopJNI.cmake Normal file
View File

@ -0,0 +1,97 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
# Common JNI detection for CMake, shared by all Native components.
#
# Check the JVM_ARCH_DATA_MODEL variable as been set to 32 or 64 by maven.
if(NOT DEFINED JVM_ARCH_DATA_MODEL)
message(FATAL_ERROR "JVM_ARCH_DATA_MODEL is not defined")
elseif(NOT (JVM_ARCH_DATA_MODEL EQUAL 32 OR JVM_ARCH_DATA_MODEL EQUAL 64))
message(FATAL_ERROR "JVM_ARCH_DATA_MODEL is not 32 or 64")
endif()
#
# Linux-specific JNI configuration.
#
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
# Locate JNI_INCLUDE_DIRS and JNI_LIBRARIES.
# Since we were invoked from Maven, we know that the JAVA_HOME environment
# variable is valid. So we ignore system paths here and just use JAVA_HOME.
file(TO_CMAKE_PATH "$ENV{JAVA_HOME}" _java_home)
if(CMAKE_SYSTEM_PROCESSOR MATCHES "^i.86$")
set(_java_libarch "i386")
elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64")
set(_java_libarch "amd64")
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^arm")
set(_java_libarch "arm")
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64le")
if(EXISTS "${_java_home}/jre/lib/ppc64le")
set(_java_libarch "ppc64le")
else()
set(_java_libarch "ppc64")
endif()
else()
set(_java_libarch ${CMAKE_SYSTEM_PROCESSOR})
endif()
set(_JDK_DIRS "${_java_home}/jre/lib/${_java_libarch}/*"
"${_java_home}/jre/lib/${_java_libarch}"
"${_java_home}/jre/lib/*"
"${_java_home}/jre/lib"
"${_java_home}/lib/*"
"${_java_home}/lib"
"${_java_home}/include/*"
"${_java_home}/include"
"${_java_home}"
)
find_path(JAVA_INCLUDE_PATH
NAMES jni.h
PATHS ${_JDK_DIRS}
NO_DEFAULT_PATH)
#In IBM java, it's jniport.h instead of jni_md.h
find_path(JAVA_INCLUDE_PATH2
NAMES jni_md.h jniport.h
PATHS ${_JDK_DIRS}
NO_DEFAULT_PATH)
set(JNI_INCLUDE_DIRS ${JAVA_INCLUDE_PATH} ${JAVA_INCLUDE_PATH2})
find_library(JAVA_JVM_LIBRARY
NAMES jvm JavaVM
PATHS ${_JDK_DIRS}
NO_DEFAULT_PATH)
set(JNI_LIBRARIES ${JAVA_JVM_LIBRARY})
unset(_java_libarch)
unset(_java_home)
message("JAVA_HOME=${JAVA_HOME}, JAVA_JVM_LIBRARY=${JAVA_JVM_LIBRARY}")
message("JAVA_INCLUDE_PATH=${JAVA_INCLUDE_PATH}, JAVA_INCLUDE_PATH2=${JAVA_INCLUDE_PATH2}")
if(JAVA_JVM_LIBRARY AND JAVA_INCLUDE_PATH AND JAVA_INCLUDE_PATH2)
message("Located all JNI components successfully.")
else()
message(FATAL_ERROR "Failed to find a viable JVM installation under JAVA_HOME.")
endif()
# Use the standard FindJNI module to locate the JNI components.
find_package(JNI REQUIRED)
#
# Otherwise, use the standard FindJNI module to locate the JNI components.
#
else()
find_package(JNI REQUIRED)
endif()

358
hadoop-common-project/hadoop-common/src/CMakeLists.txt
View File

@ -16,209 +16,149 @@
# limitations under the License.
#
#
# CMake configuration.
#
cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
# Default to release builds
set(CMAKE_BUILD_TYPE, Release)
list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/..)
include(HadoopCommon)
include(JNIFlags.cmake NO_POLICY_SCOPE)
# Compile a library with both shared and static variants
function(add_dual_library LIBNAME)
add_library(${LIBNAME} SHARED ${ARGN})
add_library(${LIBNAME}_static STATIC ${ARGN})
set_target_properties(${LIBNAME}_static PROPERTIES OUTPUT_NAME ${LIBNAME})
endfunction(add_dual_library)
# Link both a static and a dynamic target against some libraries
function(target_link_dual_libraries LIBNAME)
target_link_libraries(${LIBNAME} ${ARGN})
target_link_libraries(${LIBNAME}_static ${ARGN})
endfunction(target_link_dual_libraries)
function(output_directory TGT DIR)
SET_TARGET_PROPERTIES(${TGT} PROPERTIES
RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
SET_TARGET_PROPERTIES(${TGT} PROPERTIES
ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
SET_TARGET_PROPERTIES(${TGT} PROPERTIES
LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
endfunction(output_directory TGT DIR)
function(dual_output_directory TGT DIR)
output_directory(${TGT} "${DIR}")
output_directory(${TGT}_static "${DIR}")
endfunction(dual_output_directory TGT DIR)
# Source and test locations.
set(SRC main/native/src/org/apache/hadoop)
set(TST main/native/src/test/org/apache/hadoop)
#
# This macro alters the behavior of find_package and find_library.
# It does this by setting the CMAKE_FIND_LIBRARY_SUFFIXES global variable.
# You should save that variable before calling this function and restore it
# after you have accomplished your goal.
# Main configuration.
#
# The behavior is altered in two ways:
# 1. We always find shared libraries, never static;
# 2. We find shared libraries with the given version number.
#
# On Windows this function is a no-op. Windows does not encode
# version number information information into library path names.
#
macro(set_find_shared_library_version LVERS)
IF(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
# Mac OS uses .dylib
SET(CMAKE_FIND_LIBRARY_SUFFIXES ".${LVERS}.dylib")
ELSEIF(${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD")
# FreeBSD has always .so installed.
SET(CMAKE_FIND_LIBRARY_SUFFIXES ".so")
ELSEIF(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
# Windows doesn't support finding shared libraries by version.
ELSE()
# Most UNIX variants use .so
SET(CMAKE_FIND_LIBRARY_SUFFIXES ".so.${LVERS}")
ENDIF()
endmacro(set_find_shared_library_version LVERS)
#
# Alter the behavior of find_package and find_library so that we find only
# shared libraries without any version suffix. You should save
# CMAKE_FIND_LIBRARY_SUFFIXES before calling this function and restore it
# afterwards.
#
macro(set_find_shared_library_without_version)
IF(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
# Mac OS uses .dylib
SET(CMAKE_FIND_LIBRARY_SUFFIXES ".dylib")
ELSEIF(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
# No effect
ELSE()
# Most UNIX variants use .so
SET(CMAKE_FIND_LIBRARY_SUFFIXES ".so")
ENDIF()
endmacro(set_find_shared_library_without_version)
# The caller must specify where the generated headers have been placed.
if(NOT GENERATED_JAVAH)
message(FATAL_ERROR "You must set the CMake variable GENERATED_JAVAH")
endif()
if (NOT GENERATED_JAVAH)
# Must identify where the generated headers have been placed
MESSAGE(FATAL_ERROR "You must set the cmake variable GENERATED_JAVAH")
endif (NOT GENERATED_JAVAH)
find_package(JNI REQUIRED)
# Configure JNI.
include(HadoopJNI)
SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
set_find_shared_library_version("1")
# Require zlib.
set(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
hadoop_set_find_shared_library_version("1")
find_package(ZLIB REQUIRED)
SET(CMAKE_FIND_LIBRARY_SUFFIXES ${STORED_CMAKE_FIND_LIBRARY_SUFFIXES})
set(CMAKE_FIND_LIBRARY_SUFFIXES ${STORED_CMAKE_FIND_LIBRARY_SUFFIXES})
get_filename_component(HADOOP_ZLIB_LIBRARY ${ZLIB_LIBRARIES} NAME)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT -D_GNU_SOURCE")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64")
set(D main/native/src/org/apache/hadoop)
set(T main/native/src/test/org/apache/hadoop)
GET_FILENAME_COMPONENT(HADOOP_ZLIB_LIBRARY ${ZLIB_LIBRARIES} NAME)
SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
set_find_shared_library_version("1")
# Look for bzip2.
set(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
hadoop_set_find_shared_library_version("1")
find_package(BZip2 QUIET)
if (BZIP2_INCLUDE_DIR AND BZIP2_LIBRARIES)
GET_FILENAME_COMPONENT(HADOOP_BZIP2_LIBRARY ${BZIP2_LIBRARIES} NAME)
if(BZIP2_INCLUDE_DIR AND BZIP2_LIBRARIES)
get_filename_component(HADOOP_BZIP2_LIBRARY ${BZIP2_LIBRARIES} NAME)
set(BZIP2_SOURCE_FILES
"${D}/io/compress/bzip2/Bzip2Compressor.c"
"${D}/io/compress/bzip2/Bzip2Decompressor.c")
else (BZIP2_INCLUDE_DIR AND BZIP2_LIBRARIES)
"${SRC}/io/compress/bzip2/Bzip2Compressor.c"
"${SRC}/io/compress/bzip2/Bzip2Decompressor.c")
set(REQUIRE_BZIP2 ${REQUIRE_BZIP2}) # Stop warning about unused variable.
else()
set(BZIP2_SOURCE_FILES "")
set(BZIP2_INCLUDE_DIR "")
IF(REQUIRE_BZIP2)
MESSAGE(FATAL_ERROR "Required bzip2 library and/or header files could not be found.")
ENDIF(REQUIRE_BZIP2)
endif (BZIP2_INCLUDE_DIR AND BZIP2_LIBRARIES)
SET(CMAKE_FIND_LIBRARY_SUFFIXES ${STORED_CMAKE_FIND_LIBRARY_SUFFIXES})
if(REQUIRE_BZIP2)
message(FATAL_ERROR "Required bzip2 library and/or header files could not be found.")
endif()
endif()
set(CMAKE_FIND_LIBRARY_SUFFIXES ${STORED_CMAKE_FIND_LIBRARY_SUFFIXES})
INCLUDE(CheckFunctionExists)
INCLUDE(CheckCSourceCompiles)
INCLUDE(CheckLibraryExists)
CHECK_FUNCTION_EXISTS(sync_file_range HAVE_SYNC_FILE_RANGE)
CHECK_FUNCTION_EXISTS(posix_fadvise HAVE_POSIX_FADVISE)
CHECK_LIBRARY_EXISTS(dl dlopen "" NEED_LINK_DL)
SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
set_find_shared_library_version("1")
find_library(SNAPPY_LIBRARY
# Require snappy.
set(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
hadoop_set_find_shared_library_version("1")
find_library(SNAPPY_LIBRARY
NAMES snappy
PATHS ${CUSTOM_SNAPPY_PREFIX} ${CUSTOM_SNAPPY_PREFIX}/lib
${CUSTOM_SNAPPY_PREFIX}/lib64 ${CUSTOM_SNAPPY_LIB})
SET(CMAKE_FIND_LIBRARY_SUFFIXES ${STORED_CMAKE_FIND_LIBRARY_SUFFIXES})
find_path(SNAPPY_INCLUDE_DIR
set(CMAKE_FIND_LIBRARY_SUFFIXES ${STORED_CMAKE_FIND_LIBRARY_SUFFIXES})
find_path(SNAPPY_INCLUDE_DIR
NAMES snappy.h
PATHS ${CUSTOM_SNAPPY_PREFIX} ${CUSTOM_SNAPPY_PREFIX}/include
${CUSTOM_SNAPPY_INCLUDE})
if (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
GET_FILENAME_COMPONENT(HADOOP_SNAPPY_LIBRARY ${SNAPPY_LIBRARY} NAME)
if(SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
get_filename_component(HADOOP_SNAPPY_LIBRARY ${SNAPPY_LIBRARY} NAME)
set(SNAPPY_SOURCE_FILES
"${D}/io/compress/snappy/SnappyCompressor.c"
"${D}/io/compress/snappy/SnappyDecompressor.c")
else (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
"${SRC}/io/compress/snappy/SnappyCompressor.c"
"${SRC}/io/compress/snappy/SnappyDecompressor.c")
set(REQUIRE_SNAPPY ${REQUIRE_SNAPPY}) # Stop warning about unused variable.
message(STATUS "Found Snappy: ${SNAPPY_LIBRARY}")
else()
set(SNAPPY_INCLUDE_DIR "")
set(SNAPPY_SOURCE_FILES "")
IF(REQUIRE_SNAPPY)
MESSAGE(FATAL_ERROR "Required snappy library could not be found. SNAPPY_LIBRARY=${SNAPPY_LIBRARY}, SNAPPY_INCLUDE_DIR=${SNAPPY_INCLUDE_DIR}, CUSTOM_SNAPPY_INCLUDE_DIR=${CUSTOM_SNAPPY_INCLUDE_DIR}, CUSTOM_SNAPPY_PREFIX=${CUSTOM_SNAPPY_PREFIX}, CUSTOM_SNAPPY_INCLUDE=${CUSTOM_SNAPPY_INCLUDE}")
ENDIF(REQUIRE_SNAPPY)
endif (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
if(REQUIRE_SNAPPY)
message(FATAL_ERROR "Required snappy library could not be found. SNAPPY_LIBRARY=${SNAPPY_LIBRARY}, SNAPPY_INCLUDE_DIR=${SNAPPY_INCLUDE_DIR}, CUSTOM_SNAPPY_INCLUDE_DIR=${CUSTOM_SNAPPY_INCLUDE_DIR}, CUSTOM_SNAPPY_PREFIX=${CUSTOM_SNAPPY_PREFIX}, CUSTOM_SNAPPY_INCLUDE=${CUSTOM_SNAPPY_INCLUDE}")
endif()
endif()
IF (CMAKE_SYSTEM_PROCESSOR MATCHES "^i.86$" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64")
set(BULK_CRC_ARCH_SOURCE_FIlE "${D}/util/bulk_crc32_x86.c")
ELSEIF (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
set(BULK_CRC_ARCH_SOURCE_FIlE "${D}/util/bulk_crc32_aarch64.c")
ELSE()
MESSAGE("No HW CRC acceleration for ${CMAKE_SYSTEM_PROCESSOR}, falling back to SW")
ENDIF()
# Build hardware CRC32 acceleration, if supported on the platform.
if(CMAKE_SYSTEM_PROCESSOR MATCHES "^i.86$" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64")
set(BULK_CRC_ARCH_SOURCE_FIlE "${SRC}/util/bulk_crc32_x86.c")
elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
set(BULK_CRC_ARCH_SOURCE_FIlE "${SRC}/util/bulk_crc32_aarch64.c")
else()
message("No HW CRC acceleration for ${CMAKE_SYSTEM_PROCESSOR}, falling back to SW")
endif()
# Find the no-suffix version of libcrypto.
# See HADOOP-11216 for details.
SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
set_find_shared_library_without_version()
SET(OPENSSL_NAME "crypto")
IF(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
# Find the no-suffix version of libcrypto/openssl. See HADOOP-11216 for details.
set(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
hadoop_set_find_shared_library_without_version()
set(OPENSSL_NAME "crypto")
if(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
SET(OPENSSL_NAME "eay32")
ENDIF()
MESSAGE("CUSTOM_OPENSSL_PREFIX = ${CUSTOM_OPENSSL_PREFIX}")
endif()
message("CUSTOM_OPENSSL_PREFIX = ${CUSTOM_OPENSSL_PREFIX}")
find_library(OPENSSL_LIBRARY
NAMES ${OPENSSL_NAME}
PATHS ${CUSTOM_OPENSSL_PREFIX} ${CUSTOM_OPENSSL_PREFIX}/lib
${CUSTOM_OPENSSL_PREFIX}/lib64 ${CUSTOM_OPENSSL_LIB} NO_DEFAULT_PATH)
find_library(OPENSSL_LIBRARY NAMES ${OPENSSL_NAME})
find_path(OPENSSL_INCLUDE_DIR
find_path(OPENSSL_INCLUDE_DIR
NAMES openssl/evp.h
PATHS ${CUSTOM_OPENSSL_PREFIX} ${CUSTOM_OPENSSL_PREFIX}/include
${CUSTOM_OPENSSL_INCLUDE} NO_DEFAULT_PATH)
find_path(OPENSSL_INCLUDE_DIR NAMES openssl/evp.h)
SET(CMAKE_FIND_LIBRARY_SUFFIXES ${STORED_CMAKE_FIND_LIBRARY_SUFFIXES})
SET(USABLE_OPENSSL 0)
if (OPENSSL_LIBRARY AND OPENSSL_INCLUDE_DIR)
INCLUDE(CheckCSourceCompiles)
SET(OLD_CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES})
SET(CMAKE_REQUIRED_INCLUDES ${OPENSSL_INCLUDE_DIR})
CHECK_C_SOURCE_COMPILES("#include \"${OPENSSL_INCLUDE_DIR}/openssl/evp.h\"\nint main(int argc, char **argv) { return !EVP_aes_256_ctr; }" HAS_NEW_ENOUGH_OPENSSL)
SET(CMAKE_REQUIRED_INCLUDES ${OLD_CMAKE_REQUIRED_INCLUDES})
set(CMAKE_FIND_LIBRARY_SUFFIXES ${STORED_CMAKE_FIND_LIBRARY_SUFFIXES})
set(USABLE_OPENSSL 0)
if(OPENSSL_LIBRARY AND OPENSSL_INCLUDE_DIR)
include(CheckCSourceCompiles)
set(OLD_CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES})
set(CMAKE_REQUIRED_INCLUDES ${OPENSSL_INCLUDE_DIR})
check_c_source_compiles("#include \"${OPENSSL_INCLUDE_DIR}/openssl/evp.h\"\nint main(int argc, char **argv) { return !EVP_aes_256_ctr; }" HAS_NEW_ENOUGH_OPENSSL)
set(CMAKE_REQUIRED_INCLUDES ${OLD_CMAKE_REQUIRED_INCLUDES})
if(NOT HAS_NEW_ENOUGH_OPENSSL)
MESSAGE("The OpenSSL library installed at ${OPENSSL_LIBRARY} is too old. You need a version at least new enough to have EVP_aes_256_ctr.")
else(NOT HAS_NEW_ENOUGH_OPENSSL)
message("The OpenSSL library installed at ${OPENSSL_LIBRARY} is too old. You need a version at least new enough to have EVP_aes_256_ctr.")
else()
SET(USABLE_OPENSSL 1)
endif(NOT HAS_NEW_ENOUGH_OPENSSL)
endif (OPENSSL_LIBRARY AND OPENSSL_INCLUDE_DIR)
if (USABLE_OPENSSL)
GET_FILENAME_COMPONENT(HADOOP_OPENSSL_LIBRARY ${OPENSSL_LIBRARY} NAME)
SET(OPENSSL_SOURCE_FILES
"${D}/crypto/OpensslCipher.c"
"${D}/crypto/random/OpensslSecureRandom.c")
else (USABLE_OPENSSL)
MESSAGE("Cannot find a usable OpenSSL library. OPENSSL_LIBRARY=${OPENSSL_LIBRARY}, OPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}, CUSTOM_OPENSSL_LIB=${CUSTOM_OPENSSL_LIB}, CUSTOM_OPENSSL_PREFIX=${CUSTOM_OPENSSL_PREFIX}, CUSTOM_OPENSSL_INCLUDE=${CUSTOM_OPENSSL_INCLUDE}")
IF(REQUIRE_OPENSSL)
MESSAGE(FATAL_ERROR "Terminating build because require.openssl was specified.")
ENDIF(REQUIRE_OPENSSL)
SET(OPENSSL_LIBRARY "")
SET(OPENSSL_INCLUDE_DIR "")
SET(OPENSSL_SOURCE_FILES "")
endif (USABLE_OPENSSL)
endif()
endif()
if(USABLE_OPENSSL)
get_filename_component(HADOOP_OPENSSL_LIBRARY ${OPENSSL_LIBRARY} NAME)
set(OPENSSL_SOURCE_FILES
"${SRC}/crypto/OpensslCipher.c"
"${SRC}/crypto/random/OpensslSecureRandom.c")
set(REQUIRE_OPENSSL ${REQUIRE_OPENSSL}) # Stop warning about unused variable.
else()
message("Cannot find a usable OpenSSL library. OPENSSL_LIBRARY=${OPENSSL_LIBRARY}, OPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}, CUSTOM_OPENSSL_LIB=${CUSTOM_OPENSSL_LIB}, CUSTOM_OPENSSL_PREFIX=${CUSTOM_OPENSSL_PREFIX}, CUSTOM_OPENSSL_INCLUDE=${CUSTOM_OPENSSL_INCLUDE}")
if(REQUIRE_OPENSSL)
message(FATAL_ERROR "Terminating build because require.openssl was specified.")
endif()
set(OPENSSL_LIBRARY "")
set(OPENSSL_INCLUDE_DIR "")
set(OPENSSL_SOURCE_FILES "")
endif()
# Check for platform-specific functions and libraries.
include(CheckFunctionExists)
include(CheckLibraryExists)
check_function_exists(sync_file_range HAVE_SYNC_FILE_RANGE)
check_function_exists(posix_fadvise HAVE_POSIX_FADVISE)
check_library_exists(dl dlopen "" NEED_LINK_DL)
# Configure the build.
include_directories(
${GENERATED_JAVAH}
main/native/src
@ -230,66 +170,60 @@ include_directories(
${BZIP2_INCLUDE_DIR}
${SNAPPY_INCLUDE_DIR}
${OPENSSL_INCLUDE_DIR}
${D}/util
${SRC}/util
)
CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
configure_file(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
add_executable(test_bulk_crc32
${D}/util/bulk_crc32.c
${BULK_CRC_ARCH_SOURCE_FIlE}
${T}/util/test_bulk_crc32.c
)
SET(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
add_dual_library(hadoop
set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
hadoop_add_dual_library(hadoop
main/native/src/exception.c
${D}/io/compress/lz4/Lz4Compressor.c
${D}/io/compress/lz4/Lz4Decompressor.c
${D}/io/compress/lz4/lz4.c
${D}/io/compress/lz4/lz4hc.c
${SRC}/io/compress/lz4/Lz4Compressor.c
${SRC}/io/compress/lz4/Lz4Decompressor.c
${SRC}/io/compress/lz4/lz4.c
${SRC}/io/compress/lz4/lz4hc.c
${SNAPPY_SOURCE_FILES}
${OPENSSL_SOURCE_FILES}
${D}/io/compress/zlib/ZlibCompressor.c
${D}/io/compress/zlib/ZlibDecompressor.c
${SRC}/io/compress/zlib/ZlibCompressor.c
${SRC}/io/compress/zlib/ZlibDecompressor.c
${BZIP2_SOURCE_FILES}
${D}/io/nativeio/NativeIO.c
${D}/io/nativeio/errno_enum.c
${D}/io/nativeio/file_descriptor.c
${D}/io/nativeio/SharedFileDescriptorFactory.c
${D}/net/unix/DomainSocket.c
${D}/net/unix/DomainSocketWatcher.c
${D}/security/JniBasedUnixGroupsMapping.c
${D}/security/JniBasedUnixGroupsNetgroupMapping.c
${D}/security/hadoop_group_info.c
${D}/security/hadoop_user_info.c
${D}/util/NativeCodeLoader.c
${D}/util/NativeCrc32.c
${D}/util/bulk_crc32.c
${SRC}/io/nativeio/NativeIO.c
${SRC}/io/nativeio/errno_enum.c
${SRC}/io/nativeio/file_descriptor.c
${SRC}/io/nativeio/SharedFileDescriptorFactory.c
${SRC}/net/unix/DomainSocket.c
${SRC}/net/unix/DomainSocketWatcher.c
${SRC}/security/JniBasedUnixGroupsMapping.c
${SRC}/security/JniBasedUnixGroupsNetgroupMapping.c
${SRC}/security/hadoop_group_info.c
${SRC}/security/hadoop_user_info.c
${SRC}/util/NativeCodeLoader.c
${SRC}/util/NativeCrc32.c
${SRC}/util/bulk_crc32.c
${BULK_CRC_ARCH_SOURCE_FIlE}
)
if (NEED_LINK_DL)
if(NEED_LINK_DL)
set(LIB_DL dl)
endif (NEED_LINK_DL)
endif()
IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
#
# By embedding '$ORIGIN' into the RPATH of libhadoop.so,
# dlopen will look in the directory containing libhadoop.so.
# However, $ORIGIN is not supported by all operating systems.
#
hadoop_target_link_dual_libraries(hadoop ${LIB_DL} ${JAVA_JVM_LIBRARY})
set(LIBHADOOP_VERSION "1.0.0")
set_target_properties(hadoop PROPERTIES SOVERSION ${LIBHADOOP_VERSION})
hadoop_dual_output_directory(hadoop target/usr/local/lib)
# By embedding '$ORIGIN' into the RPATH of libhadoop.so, dlopen will look in
# the directory containing libhadoop.so. However, $ORIGIN is not supported by
# all operating systems.
if(${CMAKE_SYSTEM_NAME} MATCHES "Linux|SunOS")
set(RPATH "\$ORIGIN/")
if (EXTRA_LIBHADOOP_RPATH)
if(EXTRA_LIBHADOOP_RPATH)
set(RPATH "${RPATH}:${EXTRA_LIBHADOOP_RPATH}/")
endif(EXTRA_LIBHADOOP_RPATH)
SET_TARGET_PROPERTIES(hadoop
PROPERTIES INSTALL_RPATH "${RPATH}")
ENDIF()
endif()
set_target_properties(hadoop PROPERTIES INSTALL_RPATH "${RPATH}")
endif()
target_link_dual_libraries(hadoop
${LIB_DL}
${JAVA_JVM_LIBRARY}
# Build the CRC32 test executable.
add_executable(test_bulk_crc32
${SRC}/util/bulk_crc32.c
${BULK_CRC_ARCH_SOURCE_FIlE}
${TST}/util/test_bulk_crc32.c
)
SET(LIBHADOOP_VERSION "1.0.0")
SET_TARGET_PROPERTIES(hadoop PROPERTIES
SOVERSION ${LIBHADOOP_VERSION})
dual_output_directory(hadoop target/usr/local/lib)

61
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
View File

@ -21,6 +21,7 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.security.PrivilegedAction;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.Executors;
@ -141,6 +142,7 @@ public abstract class ZKFailoverController {
throws AccessControlException, IOException;
protected abstract InetSocketAddress getRpcAddressToBindTo();
protected abstract PolicyProvider getPolicyProvider();
protected abstract List<HAServiceTarget> getAllOtherNodes();
/**
* Return the name of a znode inside the configured parent znode in which
@ -616,9 +618,11 @@ public abstract class ZKFailoverController {
* Coordinate a graceful failover. This proceeds in several phases:
* 1) Pre-flight checks: ensure that the local node is healthy, and
* thus a candidate for failover.
* 2) Determine the current active node. If it is the local node, no
* 2a) Determine the current active node. If it is the local node, no
* need to failover - return success.
* 3) Ask that node to yield from the election for a number of seconds.
* 2b) Get the other nodes
* 3a) Ask the other nodes to yield from election for a number of seconds
* 3b) Ask the active node to yield from the election for a number of seconds.
* 4) Allow the normal election path to run in other threads. Wait until
* we either become unhealthy or we see an election attempt recorded by
* the normal code path.
@ -648,12 +652,27 @@ public abstract class ZKFailoverController {
"No need to failover. Returning success.");
return;
}
// Phase 3: ask the old active to yield from the election.
LOG.info("Asking " + oldActive + " to cede its active state for " +
timeout + "ms");
ZKFCProtocol oldZkfc = oldActive.getZKFCProxy(conf, timeout);
oldZkfc.cedeActive(timeout);
// Phase 2b: get the other nodes
List<HAServiceTarget> otherNodes = getAllOtherNodes();
List<ZKFCProtocol> otherZkfcs = new ArrayList<ZKFCProtocol>(otherNodes.size());
// Phase 3: ask the other nodes to yield from the election.
HAServiceTarget activeNode = null;
for (HAServiceTarget remote : otherNodes) {
// same location, same node - may not always be == equality
if (remote.getAddress().equals(oldActive.getAddress())) {
activeNode = remote;
continue;
}
otherZkfcs.add(cedeRemoteActive(remote, timeout));
}
assert
activeNode != null : "Active node does not match any known remote node";
// Phase 3b: ask the old active to yield
otherZkfcs.add(cedeRemoteActive(activeNode, timeout));
// Phase 4: wait for the normal election to make the local node
// active.
@ -676,8 +695,10 @@ public abstract class ZKFailoverController {
// Phase 5. At this point, we made some attempt to become active. So we
// can tell the old active to rejoin if it wants. This allows a quick
// fail-back if we immediately crash.
oldZkfc.cedeActive(-1);
for (ZKFCProtocol zkfc : otherZkfcs) {
zkfc.cedeActive(-1);
}
if (attempt.succeeded) {
LOG.info("Successfully became active. " + attempt.status);
} else {
@ -687,6 +708,23 @@ public abstract class ZKFailoverController {
}
}
/**
* Ask the remote zkfc to cede its active status and wait for the specified
* timeout before attempting to claim leader status.
* @param remote node to ask
* @param timeout amount of time to cede
* @return the {@link ZKFCProtocol} used to talk to the ndoe
* @throws IOException
*/
private ZKFCProtocol cedeRemoteActive(HAServiceTarget remote, int timeout)
throws IOException {
LOG.info("Asking " + remote + " to cede its active state for "
+ timeout + "ms");
ZKFCProtocol oldZkfc = remote.getZKFCProxy(conf, timeout);
oldZkfc.cedeActive(timeout);
return oldZkfc;
}
/**
* Ensure that the local node is in a healthy state, and thus
* eligible for graceful failover.
@ -777,7 +815,8 @@ public abstract class ZKFailoverController {
break;
default:
throw new IllegalArgumentException("Unhandled state:" + lastHealthState);
throw new IllegalArgumentException("Unhandled state:"
+ lastHealthState);
}
}
}

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
View File

@ -367,7 +367,7 @@ public class MetricsSystemImpl extends MetricsSystem implements MetricsSource {
try {
onTimerEvent();
} catch (Exception e) {
LOG.warn(e);
LOG.warn("Error invoking metrics timer", e);
}
}
}, millis, millis);

93
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/MiniZKFCCluster.java
View File

@ -22,6 +22,8 @@ import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -49,7 +51,7 @@ public class MiniZKFCCluster {
private final TestContext ctx;
private final ZooKeeperServer zks;
private DummyHAService svcs[];
private List<DummyHAService> svcs;
private DummyZKFCThread thrs[];
private Configuration conf;
@ -63,38 +65,67 @@ public class MiniZKFCCluster {
conf.setInt(CommonConfigurationKeys.HA_HM_CHECK_INTERVAL_KEY, 50);
conf.setInt(CommonConfigurationKeys.HA_HM_CONNECT_RETRY_INTERVAL_KEY, 50);
conf.setInt(CommonConfigurationKeys.HA_HM_SLEEP_AFTER_DISCONNECT_KEY, 50);
svcs = new DummyHAService[2];
svcs[0] = new DummyHAService(HAServiceState.INITIALIZING,
new InetSocketAddress("svc1", 1234));
svcs[0].setSharedResource(sharedResource);
svcs[1] = new DummyHAService(HAServiceState.INITIALIZING,
new InetSocketAddress("svc2", 1234));
svcs[1].setSharedResource(sharedResource);
svcs = new ArrayList<DummyHAService>(2);
// remove any existing instances we are keeping track of
DummyHAService.instances.clear();
for (int i = 0; i < 2; i++) {
addSvcs(svcs, i);
}
this.ctx = new TestContext();
this.zks = zks;
}
private void addSvcs(List<DummyHAService> svcs, int i) {
svcs.add(new DummyHAService(HAServiceState.INITIALIZING, new InetSocketAddress("svc" + (i + 1),
1234)));
svcs.get(i).setSharedResource(sharedResource);
}
/**
* Set up two services and their failover controllers. svc1 is started
* first, so that it enters ACTIVE state, and then svc2 is started,
* which enters STANDBY
*/
public void start() throws Exception {
start(2);
}
/**
* Set up the specified number of services and their failover controllers. svc1 is
* started first, so that it enters ACTIVE state, and then svc2...svcN is started, which enters
* STANDBY.
* <p>
* Adds any extra svc needed beyond the first two before starting the rest of the cluster.
* @param count number of zkfcs to start
*/
public void start(int count) throws Exception {
// setup the expected number of zkfcs, if we need to add more. This seemed the least invasive
// way to add the services - otherwise its a large test rewrite or changing a lot of assumptions
if (count > 2) {
for (int i = 2; i < count; i++) {
addSvcs(svcs, i);
}
}
// Format the base dir, should succeed
thrs = new DummyZKFCThread[2];
thrs[0] = new DummyZKFCThread(ctx, svcs[0]);
thrs = new DummyZKFCThread[count];
thrs[0] = new DummyZKFCThread(ctx, svcs.get(0));
assertEquals(0, thrs[0].zkfc.run(new String[]{"-formatZK"}));
ctx.addThread(thrs[0]);
thrs[0].start();
LOG.info("Waiting for svc0 to enter active state");
waitForHAState(0, HAServiceState.ACTIVE);
LOG.info("Adding svc1");
thrs[1] = new DummyZKFCThread(ctx, svcs[1]);
thrs[1].start();
waitForHAState(1, HAServiceState.STANDBY);
// add the remaining zkfc
for (int i = 1; i < count; i++) {
LOG.info("Adding svc" + i);
thrs[i] = new DummyZKFCThread(ctx, svcs.get(i));
thrs[i].start();
waitForHAState(i, HAServiceState.STANDBY);
}
}
/**
@ -122,7 +153,7 @@ public class MiniZKFCCluster {
}
public DummyHAService getService(int i) {
return svcs[i];
return svcs.get(i);
}
public ActiveStandbyElector getElector(int i) {
@ -134,23 +165,23 @@ public class MiniZKFCCluster {
}
public void setHealthy(int idx, boolean healthy) {
svcs[idx].isHealthy = healthy;
svcs.get(idx).isHealthy = healthy;
}
public void setFailToBecomeActive(int idx, boolean doFail) {
svcs[idx].failToBecomeActive = doFail;
svcs.get(idx).failToBecomeActive = doFail;
}
public void setFailToBecomeStandby(int idx, boolean doFail) {
svcs[idx].failToBecomeStandby = doFail;
svcs.get(idx).failToBecomeStandby = doFail;
}
public void setFailToFence(int idx, boolean doFail) {
svcs[idx].failToFence = doFail;
svcs.get(idx).failToFence = doFail;
}
public void setUnreachable(int idx, boolean unreachable) {
svcs[idx].actUnreachable = unreachable;
svcs.get(idx).actUnreachable = unreachable;
}
/**
@ -204,7 +235,7 @@ public class MiniZKFCCluster {
byte[] data = zks.getZKDatabase().getData(
DummyZKFC.LOCK_ZNODE, stat, null);
assertArrayEquals(Ints.toByteArray(svcs[idx].index), data);
assertArrayEquals(Ints.toByteArray(svcs.get(idx).index), data);
long session = stat.getEphemeralOwner();
LOG.info("Expiring svc " + idx + "'s zookeeper session " + session);
zks.closeSession(session);
@ -218,7 +249,7 @@ public class MiniZKFCCluster {
*/
public void waitForActiveLockHolder(Integer idx)
throws Exception {
DummyHAService svc = idx == null ? null : svcs[idx];
DummyHAService svc = idx == null ? null : svcs.get(idx);
ActiveStandbyElectorTestUtil.waitForActiveLockData(ctx, zks,
DummyZKFC.SCOPED_PARENT_ZNODE,
(idx == null) ? null : Ints.toByteArray(svc.index));
@ -320,5 +351,17 @@ public class MiniZKFCCluster {
protected PolicyProvider getPolicyProvider() {
return null;
}
@Override
protected List<HAServiceTarget> getAllOtherNodes() {
List<HAServiceTarget> services = new ArrayList<HAServiceTarget>(
DummyHAService.instances.size());
for (DummyHAService service : DummyHAService.instances) {
if (service != this.localTarget) {
services.add(service);
}
}
return services;
}
}
}

32
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestZKFailoverController.java
View File

@ -605,6 +605,38 @@ public class TestZKFailoverController extends ClientBaseWithFixes {
}
}
@Test(timeout = 25000)
public void testGracefulFailoverMultipleZKfcs() throws Exception {
try {
cluster.start(3);
cluster.waitForActiveLockHolder(0);
// failover to first
cluster.getService(1).getZKFCProxy(conf, 5000).gracefulFailover();
cluster.waitForActiveLockHolder(1);
// failover to second
cluster.getService(2).getZKFCProxy(conf, 5000).gracefulFailover();
cluster.waitForActiveLockHolder(2);
// failover back to original
cluster.getService(0).getZKFCProxy(conf, 5000).gracefulFailover();
cluster.waitForActiveLockHolder(0);
Thread.sleep(10000); // allow to quiesce
assertEquals(0, cluster.getService(0).fenceCount);
assertEquals(0, cluster.getService(1).fenceCount);
assertEquals(0, cluster.getService(2).fenceCount);
assertEquals(2, cluster.getService(0).activeTransitionCount);
assertEquals(1, cluster.getService(1).activeTransitionCount);
assertEquals(1, cluster.getService(2).activeTransitionCount);
} finally {
cluster.stop();
}
}
private int runFC(DummyHAService target, String ... args) throws Exception {
DummyZKFC zkfc = new DummyZKFC(conf, target);
return zkfc.run(args);

187
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestAuthenticationSessionCookie.java Normal file
View File

@ -0,0 +1,187 @@
/**
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License. See accompanying LICENSE file.
*/
package org.apache.hadoop.http;
import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
import org.junit.After;
import org.junit.Test;
import org.mortbay.log.Log;
import javax.servlet.*;
import javax.servlet.http.HttpServletResponse;
import java.io.File;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
import java.net.HttpCookie;
import java.util.List;
public class TestAuthenticationSessionCookie {
private static final String BASEDIR = System.getProperty("test.build.dir",
"target/test-dir") + "/" + TestHttpCookieFlag.class.getSimpleName();
private static boolean isCookiePersistent;
private static final long TOKEN_VALIDITY_SEC = 1000;
private static long expires;
private static String keystoresDir;
private static String sslConfDir;
private static HttpServer2 server;
public static class DummyAuthenticationFilter implements Filter {
@Override
public void init(FilterConfig filterConfig) throws ServletException {
isCookiePersistent = false;
}
@Override
public void doFilter(ServletRequest request, ServletResponse response,
FilterChain chain) throws IOException,
ServletException {
HttpServletResponse resp = (HttpServletResponse) response;
AuthenticationFilter.createAuthCookie(resp, "token", null, null, expires,
isCookiePersistent, true);
chain.doFilter(request, resp);
}
@Override
public void destroy() {
}
}
public static class DummyFilterInitializer extends FilterInitializer {
@Override
public void initFilter(FilterContainer container, Configuration conf) {
container.addFilter("DummyAuth", DummyAuthenticationFilter.class
.getName(), null);
}
}
public static class Dummy2AuthenticationFilter
extends DummyAuthenticationFilter {
@Override
public void init(FilterConfig filterConfig) throws ServletException {
isCookiePersistent = true;
expires = System.currentTimeMillis() + TOKEN_VALIDITY_SEC;
}
@Override
public void destroy() {
}
}
public static class Dummy2FilterInitializer extends FilterInitializer {
@Override
public void initFilter(FilterContainer container, Configuration conf) {
container.addFilter("Dummy2Auth", Dummy2AuthenticationFilter.class
.getName(), null);
}
}
public void startServer(boolean isTestSessionCookie) throws Exception {
Configuration conf = new Configuration();
if (isTestSessionCookie) {
conf.set(HttpServer2.FILTER_INITIALIZER_PROPERTY,
DummyFilterInitializer.class.getName());
} else {
conf.set(HttpServer2.FILTER_INITIALIZER_PROPERTY,
Dummy2FilterInitializer.class.getName());
}
File base = new File(BASEDIR);
FileUtil.fullyDelete(base);
base.mkdirs();
keystoresDir = new File(BASEDIR).getAbsolutePath();
sslConfDir = KeyStoreTestUtil.getClasspathDir(TestSSLHttpServer.class);
KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false);
Configuration sslConf = new Configuration(false);
sslConf.addResource("ssl-server.xml");
sslConf.addResource("ssl-client.xml");
server = new HttpServer2.Builder()
.setName("test")
.addEndpoint(new URI("http://localhost"))
.addEndpoint(new URI("https://localhost"))
.setConf(conf)
.keyPassword(sslConf.get("ssl.server.keystore.keypassword"))
.keyStore(sslConf.get("ssl.server.keystore.location"),
sslConf.get("ssl.server.keystore.password"),
sslConf.get("ssl.server.keystore.type", "jks"))
.trustStore(sslConf.get("ssl.server.truststore.location"),
sslConf.get("ssl.server.truststore.password"),
sslConf.get("ssl.server.truststore.type", "jks")).build();
server.addServlet("echo", "/echo", TestHttpServer.EchoServlet.class);
server.start();
}
@Test
public void testSessionCookie() throws IOException {
try {
startServer(true);
} catch (Exception e) {
// Auto-generated catch block
e.printStackTrace();
}
URL base = new URL("http://" + NetUtils.getHostPortString(server
.getConnectorAddress(0)));
HttpURLConnection conn = (HttpURLConnection) new URL(base,
"/echo").openConnection();
String header = conn.getHeaderField("Set-Cookie");
List<HttpCookie> cookies = HttpCookie.parse(header);
Assert.assertTrue(!cookies.isEmpty());
Log.info(header);
Assert.assertFalse(header.contains("; Expires="));
Assert.assertTrue("token".equals(cookies.get(0).getValue()));
}
@Test
public void testPersistentCookie() throws IOException {
try {
startServer(false);
} catch (Exception e) {
// Auto-generated catch block
e.printStackTrace();
}
URL base = new URL("http://" + NetUtils.getHostPortString(server
.getConnectorAddress(0)));
HttpURLConnection conn = (HttpURLConnection) new URL(base,
"/echo").openConnection();
String header = conn.getHeaderField("Set-Cookie");
List<HttpCookie> cookies = HttpCookie.parse(header);
Assert.assertTrue(!cookies.isEmpty());
Log.info(header);
Assert.assertTrue(header.contains("; Expires="));
Assert.assertTrue("token".equals(cookies.get(0).getValue()));
}
@After
public void cleanup() throws Exception {
server.stop();
FileUtil.fullyDelete(new File(BASEDIR));
KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir);
}
}

2
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpCookieFlag.java
View File

@ -60,7 +60,7 @@ public class TestHttpCookieFlag {
HttpServletResponse resp = (HttpServletResponse) response;
boolean isHttps = "https".equals(request.getScheme());
AuthenticationFilter.createAuthCookie(resp, "token", null, null, -1,
isHttps);
true, isHttps);
chain.doFilter(request, resp);
}

10
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
View File

@ -35,6 +35,16 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>

7
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
View File

@ -317,10 +317,9 @@ public class DFSUtilClient {
if (address != null) {
InetSocketAddress isa = NetUtils.createSocketAddr(address);
if (isa.isUnresolved()) {
LOG.warn("Namenode for " + nsId +
" remains unresolved for ID " + nnId +
". Check your hdfs-site.xml file to " +
"ensure namenodes are configured properly.");
LOG.warn("Namenode for {} remains unresolved for ID {}. Check your "
+ "hdfs-site.xml file to ensure namenodes are configured "
+ "properly.", nsId, nnId);
}
ret.put(nnId, isa);
}

12
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/BlockStoragePolicy.java
View File

@ -158,13 +158,11 @@ public class BlockStoragePolicy implements BlockStoragePolicySpi {
// remove excess storage types after fallback replacement.
diff(storageTypes, excess, null);
if (storageTypes.size() < expectedSize) {
LOG.warn("Failed to place enough replicas: expected size is " + expectedSize
+ " but only " + storageTypes.size() + " storage types can be selected "
+ "(replication=" + replication
+ ", selected=" + storageTypes
+ ", unavailable=" + unavailables
+ ", removed=" + removed
+ ", policy=" + this + ")");
LOG.warn("Failed to place enough replicas: expected size is {}"
+ " but only {} storage types can be selected (replication={},"
+ " selected={}, unavailable={}" + ", removed={}" + ", policy={}"
+ ")", expectedSize, storageTypes.size(), replication, storageTypes,
unavailables, removed, this);
}
return storageTypes;
}

3
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolInfo.java
View File

@ -24,8 +24,6 @@ import javax.annotation.Nullable;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.InvalidRequestException;
@ -41,7 +39,6 @@ import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo.Expiration;
@InterfaceAudience.Public
@InterfaceStability.Evolving
public class CachePoolInfo {
public static final Log LOG = LogFactory.getLog(CachePoolInfo.class);
/**
* Indicates that the pool does not have a maximum relative expiry.

3
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
View File

@ -853,7 +853,8 @@ public interface ClientProtocol {
/**
* Rolling upgrade operations.
* @param action either query, prepare or finalize.
* @return rolling upgrade information.
* @return rolling upgrade information. On query, if no upgrade is in
* progress, returns null.
*/
@Idempotent
public RollingUpgradeInfo rollingUpgrade(RollingUpgradeAction action)

16
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/TokenAspect.java
View File

@ -21,8 +21,6 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URI;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.DelegationTokenRenewer;
@ -37,6 +35,8 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenRenewer;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
@ -115,11 +115,11 @@ final class TokenAspect<T extends FileSystem & Renewable> {
private final DTSelecorByKind dtSelector;
private final T fs;
private boolean hasInitedToken;
private final Log LOG;
private final Logger LOG;
private final Text serviceName;
TokenAspect(T fs, final Text serviceName, final Text kind) {
this.LOG = LogFactory.getLog(fs.getClass());
this.LOG = LoggerFactory.getLogger(fs.getClass());
this.fs = fs;
this.dtSelector = new DTSelecorByKind(kind);
this.serviceName = serviceName;
@ -134,8 +134,8 @@ final class TokenAspect<T extends FileSystem & Renewable> {
if (token != null) {
fs.setDelegationToken(token);
addRenewAction(fs);
if(LOG.isDebugEnabled()) {
LOG.debug("Created new DT for " + token.getService());
if (LOG.isDebugEnabled()) {
LOG.debug("Created new DT for {}", token.getService());
}
}
hasInitedToken = true;
@ -149,8 +149,8 @@ final class TokenAspect<T extends FileSystem & Renewable> {
synchronized void initDelegationToken(UserGroupInformation ugi) {
Token<?> token = selectDelegationToken(ugi);
if (token != null) {
if(LOG.isDebugEnabled()) {
LOG.debug("Found existing DT for " + token.getService());
if (LOG.isDebugEnabled()) {
LOG.debug("Found existing DT for {}", token.getService());
}
fs.setDelegationToken(token);
hasInitedToken = true;

13
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java
View File

@ -28,8 +28,6 @@ import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLSocketFactory;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@ -38,6 +36,8 @@ import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
@ -47,7 +47,8 @@ import com.google.common.annotations.VisibleForTesting;
@InterfaceAudience.LimitedPrivate({ "HDFS" })
@InterfaceStability.Unstable
public class URLConnectionFactory {
private static final Log LOG = LogFactory.getLog(URLConnectionFactory.class);
private static final Logger LOG = LoggerFactory
.getLogger(URLConnectionFactory.class);
/**
* Timeout for socket connects and reads
@ -154,16 +155,14 @@ public class URLConnectionFactory {
throws IOException, AuthenticationException {
if (isSpnego) {
if (LOG.isDebugEnabled()) {
LOG.debug("open AuthenticatedURL connection" + url);
LOG.debug("open AuthenticatedURL connection {}", url);
}
UserGroupInformation.getCurrentUser().checkTGTAndReloginFromKeytab();
final AuthenticatedURL.Token authToken = new AuthenticatedURL.Token();
return new AuthenticatedURL(new KerberosUgiAuthenticator(),
connConfigurator).openConnection(url, authToken);
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("open URL connection");
}
LOG.debug("open URL connection");
URLConnection connection = url.openConnection();
if (connection instanceof HttpURLConnection) {
connConfigurator.configure((HttpURLConnection) connection);

27
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
View File

@ -36,8 +36,6 @@ import java.util.StringTokenizer;
import javax.ws.rs.core.MediaType;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.CommonConfigurationKeys;
@ -81,6 +79,8 @@ import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelect
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.StringUtils;
import org.codehaus.jackson.map.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@ -89,7 +89,8 @@ import com.google.common.collect.Lists;
/** A FileSystem for HDFS over the web. */
public class WebHdfsFileSystem extends FileSystem
implements DelegationTokenRenewer.Renewable, TokenAspect.TokenManagementDelegator {
public static final Log LOG = LogFactory.getLog(WebHdfsFileSystem.class);
public static final Logger LOG = LoggerFactory
.getLogger(WebHdfsFileSystem.class);
/** WebHdfs version. */
public static final int VERSION = 1;
/** Http URI: http://namenode:port/{PATH_PREFIX}/path/to/file */
@ -221,14 +222,14 @@ public class WebHdfsFileSystem extends FileSystem
// to get another token to match hdfs/rpc behavior
if (token != null) {
if(LOG.isDebugEnabled()) {
LOG.debug("Using UGI token: " + token);
LOG.debug("Using UGI token: {}", token);
}
canRefreshDelegationToken = false;
} else {
token = getDelegationToken(null);
if (token != null) {
if(LOG.isDebugEnabled()) {
LOG.debug("Fetched new token: " + token);
LOG.debug("Fetched new token: {}", token);
}
} else { // security is disabled
canRefreshDelegationToken = false;
@ -245,7 +246,7 @@ public class WebHdfsFileSystem extends FileSystem
if (canRefreshDelegationToken) {
Token<?> token = getDelegationToken(null);
if(LOG.isDebugEnabled()) {
LOG.debug("Replaced expired token: " + token);
LOG.debug("Replaced expired token: {}", token);
}
setDelegationToken(token);
replaced = (token != null);
@ -430,7 +431,7 @@ public class WebHdfsFileSystem extends FileSystem
final URL url = new URL(getTransportScheme(), nnAddr.getHostName(),
nnAddr.getPort(), path + '?' + query);
if (LOG.isTraceEnabled()) {
LOG.trace("url=" + url);
LOG.trace("url={}", url);
}
return url;
}
@ -467,7 +468,7 @@ public class WebHdfsFileSystem extends FileSystem
+ Param.toSortedString("&", parameters);
final URL url = getNamenodeURL(path, query);
if (LOG.isTraceEnabled()) {
LOG.trace("url=" + url);
LOG.trace("url={}", url);
}
return url;
}
@ -658,9 +659,9 @@ public class WebHdfsFileSystem extends FileSystem
a.action == RetryPolicy.RetryAction.RetryDecision.FAILOVER_AND_RETRY;
if (isRetry || isFailoverAndRetry) {
LOG.info("Retrying connect to namenode: " + nnAddr
+ ". Already tried " + retry + " time(s); retry policy is "
+ retryPolicy + ", delay " + a.delayMillis + "ms.");
LOG.info("Retrying connect to namenode: {}. Already tried {}"
+ " time(s); retry policy is {}, delay {}ms.", nnAddr, retry,
retryPolicy, a.delayMillis);
if (isFailoverAndRetry) {
resetStateToFailOver();
@ -757,7 +758,7 @@ public class WebHdfsFileSystem extends FileSystem
final IOException ioe =
new IOException("Response decoding failure: "+e.toString(), e);
if (LOG.isDebugEnabled()) {
LOG.debug(ioe);
LOG.debug("Response decoding failure: {}", e.toString(), e);
}
throw ioe;
} finally {
@ -1212,7 +1213,7 @@ public class WebHdfsFileSystem extends FileSystem
}
} catch (IOException ioe) {
if (LOG.isDebugEnabled()) {
LOG.debug("Token cancel failed: " + ioe);
LOG.debug("Token cancel failed: ", ioe);
}
} finally {
super.close();

30
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -38,6 +38,8 @@ Trunk (Unreleased)
HDFS-3125. Add JournalService to enable Journal Daemon. (suresh)
HDFS-6440. Support more than 2 NameNodes. (Jesse Yates via atm)
IMPROVEMENTS
HDFS-4665. Move TestNetworkTopologyWithNodeGroup to common.
@ -658,6 +660,28 @@ Release 2.8.0 - UNRELEASED
HDFS-8192. Eviction should key off used locked memory instead of
ram disk free space. (Arpit Agarwal)
HDFS-6564. Use slf4j instead of common-logging in hdfs-client.
(Rakesh R via wheat9)
HDFS-8639. Add Option for NameNode HTTP port in MiniDFSClusterManager.
(Kai Sasaki via jing9)
HDFS-8462. Implement GETXATTRS and LISTXATTRS operations for WebImageViewer.
(Jagadesh Kiran N via aajisaka)
HDFS-8640. Make reserved RBW space visible through JMX. (kanaka kumar
avvaru via Arpit Agarwal)
HDFS-8665. Fix replication check in DFSTestUtils#waitForReplication. (wang)
HDFS-8546. Use try with resources in DataStorage and Storage. (wang)
HDFS-8651. Make hadoop-hdfs-project Native code -Wall-clean (Alan Burlison
via Colin P. McCabe)
HDFS-8623. Refactor NameNode handling of invalid, corrupt, and under-recovery
blocks. (Zhe Zhang via jing9)
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
@ -941,6 +965,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8542. WebHDFS getHomeDirectory behavior does not match specification.
(Kanaka Kumar Avvaru via jghoman)
HDFS-8546. Prune cached replicas from DatanodeDescriptor state on replica
invalidation. (wang)
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
@ -1083,6 +1110,9 @@ Release 2.7.1 - UNRELEASED
HDFS-8626. Reserved RBW space is not released if creation of RBW File
fails. (kanaka kumar avvaru via Arpit Agarwal)
HDFS08656. Preserve compatibility of ClientProtocol#rollingUpgrade after
finalization. (wang)
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES

7
hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperHACheckpoints.java
View File

@ -32,6 +32,10 @@ import org.junit.BeforeClass;
* using a bookkeeper journal manager as the shared directory
*/
public class TestBookKeeperHACheckpoints extends TestStandbyCheckpoints {
//overwrite the nn count
static{
TestStandbyCheckpoints.NUM_NNS = 2;
}
private static BKJMUtil bkutil = null;
static int numBookies = 3;
static int journalCount = 0;
@ -57,8 +61,7 @@ public class TestBookKeeperHACheckpoints extends TestStandbyCheckpoints {
.build();
cluster.waitActive();
nn0 = cluster.getNameNode(0);
nn1 = cluster.getNameNode(1);
setNNs();
fs = HATestUtil.configureFailoverFs(cluster, conf);
cluster.transitionToActive(0);

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
View File

@ -132,6 +132,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_DEFAULT = "0.0.0.0:50090";
public static final String DFS_NAMENODE_SECONDARY_HTTPS_ADDRESS_KEY = "dfs.namenode.secondary.https-address";
public static final String DFS_NAMENODE_SECONDARY_HTTPS_ADDRESS_DEFAULT = "0.0.0.0:50091";
public static final String DFS_NAMENODE_CHECKPOINT_QUIET_MULTIPLIER_KEY = "dfs.namenode.checkpoint.check.quiet-multiplier";
public static final double DFS_NAMENODE_CHECKPOINT_QUIET_MULTIPLIER_DEFAULT = 1.5;
public static final String DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_KEY = "dfs.namenode.checkpoint.check.period";
public static final long DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_DEFAULT = 60;
public static final String DFS_NAMENODE_CHECKPOINT_PERIOD_KEY = "dfs.namenode.checkpoint.period";
@ -544,6 +546,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final int DFS_HA_LOGROLL_PERIOD_DEFAULT = 2 * 60; // 2m
public static final String DFS_HA_TAILEDITS_PERIOD_KEY = "dfs.ha.tail-edits.period";
public static final int DFS_HA_TAILEDITS_PERIOD_DEFAULT = 60; // 1m
public static final String DFS_HA_TAILEDITS_ALL_NAMESNODES_RETRY_KEY = "dfs.ha.tail-edits.namenode-retries";
public static final int DFS_HA_TAILEDITS_ALL_NAMESNODES_RETRY_DEFAULT = 3;
public static final String DFS_HA_LOGROLL_RPC_TIMEOUT_KEY = "dfs.ha.log-roll.rpc.timeout";
public static final int DFS_HA_LOGROLL_RPC_TIMEOUT_DEFAULT = 20000; // 20s
public static final String DFS_HA_FENCE_METHODS_KEY = "dfs.ha.fencing.methods";

36
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java
View File

@ -143,7 +143,7 @@ public class HAUtil {
* @param conf the configuration of this node
* @return the NN ID of the other node in this nameservice
*/
public static String getNameNodeIdOfOtherNode(Configuration conf, String nsId) {
public static List<String> getNameNodeIdOfOtherNodes(Configuration conf, String nsId) {
Preconditions.checkArgument(nsId != null,
"Could not determine namespace id. Please ensure that this " +
"machine is one of the machines listed as a NN RPC address, " +
@ -157,20 +157,20 @@ public class HAUtil {
DFSUtil.addKeySuffixes(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX,
nsId),
nsId);
Preconditions.checkArgument(nnIds.size() == 2,
"Expected exactly 2 NameNodes in namespace '%s'. " +
"Instead, got only %s (NN ids were '%s'",
nsId, nnIds.size(), Joiner.on("','").join(nnIds));
Preconditions.checkArgument(nnIds.size() >= 2,
"Expected at least 2 NameNodes in namespace '%s'. " +
"Instead, got only %s (NN ids were '%s')",
nsId, nnIds.size(), Joiner.on("','").join(nnIds));
Preconditions.checkState(myNNId != null && !myNNId.isEmpty(),
"Could not determine own NN ID in namespace '%s'. Please " +
"ensure that this node is one of the machines listed as an " +
"NN RPC address, or configure " + DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY,
nsId);
ArrayList<String> nnSet = Lists.newArrayList(nnIds);
nnSet.remove(myNNId);
assert nnSet.size() == 1;
return nnSet.get(0);
ArrayList<String> namenodes = Lists.newArrayList(nnIds);
namenodes.remove(myNNId);
assert namenodes.size() >= 1;
return namenodes;
}
/**
@ -180,16 +180,20 @@ public class HAUtil {
* @param myConf the configuration of this node
* @return the configuration of the other node in an HA setup
*/
public static Configuration getConfForOtherNode(
public static List<Configuration> getConfForOtherNodes(
Configuration myConf) {
String nsId = DFSUtil.getNamenodeNameServiceId(myConf);
String otherNn = getNameNodeIdOfOtherNode(myConf, nsId);
// Look up the address of the active NN.
Configuration confForOtherNode = new Configuration(myConf);
NameNode.initializeGenericKeys(confForOtherNode, nsId, otherNn);
return confForOtherNode;
List<String> otherNn = getNameNodeIdOfOtherNodes(myConf, nsId);
// Look up the address of the other NNs
List<Configuration> confs = new ArrayList<Configuration>(otherNn.size());
for (String nn : otherNn) {
Configuration confForOtherNode = new Configuration(myConf);
NameNode.initializeGenericKeys(confForOtherNode, nsId, nn);
confs.add(confForOtherNode);
}
return confs;
}
/**

40
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
View File

@ -52,17 +52,11 @@ import com.google.common.base.Preconditions;
@InterfaceAudience.Private
public class BlockTokenSecretManager extends
SecretManager<BlockTokenIdentifier> {
public static final Log LOG = LogFactory
.getLog(BlockTokenSecretManager.class);
// We use these in an HA setup to ensure that the pair of NNs produce block
// token serial numbers that are in different ranges.
private static final int LOW_MASK = ~(1 << 31);
public static final Log LOG = LogFactory.getLog(BlockTokenSecretManager.class);
public static final Token<BlockTokenIdentifier> DUMMY_TOKEN = new Token<BlockTokenIdentifier>();
private final boolean isMaster;
private int nnIndex;
/**
* keyUpdateInterval is the interval that NN updates its block keys. It should
@ -77,21 +71,22 @@ public class BlockTokenSecretManager extends
private final Map<Integer, BlockKey> allKeys;
private String blockPoolId;
private final String encryptionAlgorithm;
private final int intRange;
private final int nnRangeStart;
private final SecureRandom nonceGenerator = new SecureRandom();
;
/**
* Constructor for slaves.
*
*
* @param keyUpdateInterval how often a new key will be generated
* @param tokenLifetime how long an individual token is valid
*/
public BlockTokenSecretManager(long keyUpdateInterval,
long tokenLifetime, String blockPoolId, String encryptionAlgorithm) {
this(false, keyUpdateInterval, tokenLifetime, blockPoolId,
encryptionAlgorithm);
encryptionAlgorithm, 0, 1);
}
/**
@ -99,23 +94,25 @@ public class BlockTokenSecretManager extends
*
* @param keyUpdateInterval how often a new key will be generated
* @param tokenLifetime how long an individual token is valid
* @param nnIndex namenode index
* @param nnIndex namenode index of the namenode for which we are creating the manager
* @param blockPoolId block pool ID
* @param encryptionAlgorithm encryption algorithm to use
* @param numNNs number of namenodes possible
*/
public BlockTokenSecretManager(long keyUpdateInterval,
long tokenLifetime, int nnIndex, String blockPoolId,
long tokenLifetime, int nnIndex, int numNNs, String blockPoolId,
String encryptionAlgorithm) {
this(true, keyUpdateInterval, tokenLifetime, blockPoolId,
encryptionAlgorithm);
Preconditions.checkArgument(nnIndex == 0 || nnIndex == 1);
this.nnIndex = nnIndex;
this(true, keyUpdateInterval, tokenLifetime, blockPoolId, encryptionAlgorithm, nnIndex, numNNs);
Preconditions.checkArgument(nnIndex >= 0);
Preconditions.checkArgument(numNNs > 0);
setSerialNo(new SecureRandom().nextInt());
generateKeys();
}
private BlockTokenSecretManager(boolean isMaster, long keyUpdateInterval,
long tokenLifetime, String blockPoolId, String encryptionAlgorithm) {
long tokenLifetime, String blockPoolId, String encryptionAlgorithm, int nnIndex, int numNNs) {
this.intRange = Integer.MAX_VALUE / numNNs;
this.nnRangeStart = intRange * nnIndex;
this.isMaster = isMaster;
this.keyUpdateInterval = keyUpdateInterval;
this.tokenLifetime = tokenLifetime;
@ -127,7 +124,8 @@ public class BlockTokenSecretManager extends
@VisibleForTesting
public synchronized void setSerialNo(int serialNo) {
this.serialNo = (serialNo & LOW_MASK) | (nnIndex << 31);
// we mod the serial number by the range and then add that times the index
this.serialNo = (serialNo % intRange) + (nnRangeStart);
}
public void setBlockPoolId(String blockPoolId) {

12
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
View File

@ -172,19 +172,23 @@ public abstract class BlockInfo extends Block
public abstract int numNodes();
/**
* Add a {@link DatanodeStorageInfo} location for a block.
* Add a {@link DatanodeStorageInfo} location for a block
* @param storage The storage to add
* @param reportedBlock The block reported from the datanode. This is only
* used by erasure coded blocks, this block's id contains
* information indicating the index of the block in the
* corresponding block group.
*/
abstract boolean addStorage(DatanodeStorageInfo storage);
abstract boolean addStorage(DatanodeStorageInfo storage, Block reportedBlock);
/**
* Remove {@link DatanodeStorageInfo} location for a block
*/
abstract boolean removeStorage(DatanodeStorageInfo storage);
/**
* Replace the current BlockInfo with the new one in corresponding
* DatanodeStorageInfo's linked list
* DatanodeStorageInfo's linked list.
*/
abstract void replaceBlock(BlockInfo newBlock);

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
View File

@ -45,7 +45,7 @@ public class BlockInfoContiguous extends BlockInfo {
}
@Override
boolean addStorage(DatanodeStorageInfo storage) {
boolean addStorage(DatanodeStorageInfo storage, Block reportedBlock) {
return ContiguousBlockStorageOp.addStorage(this, storage);
}

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstructionContiguous.java
View File

@ -69,7 +69,7 @@ public class BlockInfoUnderConstructionContiguous extends
}
@Override
boolean addStorage(DatanodeStorageInfo storage) {
boolean addStorage(DatanodeStorageInfo storage, Block reportedBlock) {
return ContiguousBlockStorageOp.addStorage(this, storage);
}

626
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
View File

File diff suppressed because it is too large Load Diff

15
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
View File

@ -24,6 +24,7 @@ import java.util.List;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage.State;
@ -226,7 +227,7 @@ public class DatanodeStorageInfo {
return blockPoolUsed;
}
public AddBlockResult addBlock(BlockInfo b) {
public AddBlockResult addBlock(BlockInfo b, Block reportedBlock) {
// First check whether the block belongs to a different storage
// on the same DN.
AddBlockResult result = AddBlockResult.ADDED;
@ -245,10 +246,18 @@ public class DatanodeStorageInfo {
}
// add to the head of the data-node list
b.addStorage(this);
b.addStorage(this, reportedBlock);
insertToList(b);
return result;
}
AddBlockResult addBlock(BlockInfo b) {
return addBlock(b, b);
}
public void insertToList(BlockInfo b) {
blockList = b.listInsert(blockList, this);
numBlocks++;
return result;
}
public boolean removeBlock(BlockInfo b) {

24
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
View File

@ -709,6 +709,7 @@ public abstract class Storage extends StorageInfo {
try {
res = file.getChannel().tryLock();
if (null == res) {
LOG.error("Unable to acquire file lock on path " + lockF.toString());
throw new OverlappingFileLockException();
}
file.write(jvmName.getBytes(Charsets.UTF_8));
@ -972,35 +973,28 @@ public abstract class Storage extends StorageInfo {
public void writeProperties(File to, StorageDirectory sd) throws IOException {
Properties props = new Properties();
setPropertiesFromFields(props, sd);
writeProperties(to, sd, props);
writeProperties(to, props);
}
public static void writeProperties(File to, StorageDirectory sd,
Properties props) throws IOException {
RandomAccessFile file = new RandomAccessFile(to, "rws");
FileOutputStream out = null;
try {
public static void writeProperties(File to, Properties props)
throws IOException {
try (RandomAccessFile file = new RandomAccessFile(to, "rws");
FileOutputStream out = new FileOutputStream(file.getFD())) {
file.seek(0);
out = new FileOutputStream(file.getFD());
/*
* If server is interrupted before this line,
* If server is interrupted before this line,
* the version file will remain unchanged.
*/
props.store(out, null);
/*
* Now the new fields are flushed to the head of the file, but file
* length can still be larger then required and therefore the file can
* Now the new fields are flushed to the head of the file, but file
* length can still be larger then required and therefore the file can
* contain whole or corrupted fields from its old contents in the end.
* If server is interrupted here and restarted later these extra fields
* either should not effect server behavior or should be handled
* by the server correctly.
*/
file.setLength(out.getChannel().position());
} finally {
if (out != null) {
out.close();
}
file.close();
}
}

6
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
View File

@ -709,8 +709,10 @@ class BPServiceActor implements Runnable {
}
processCommand(cmds == null ? null : cmds.toArray(new DatanodeCommand[cmds.size()]));
DatanodeCommand cmd = cacheReport();
processCommand(new DatanodeCommand[]{ cmd });
if (!dn.areCacheReportsDisabledForTests()) {
DatanodeCommand cmd = cacheReport();
processCommand(new DatanodeCommand[]{ cmd });
}
//
// There is no work to do; sleep until hearbeat timer elapses,

17
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
View File

@ -301,6 +301,7 @@ public class DataNode extends ReconfigurableBase
ThreadGroup threadGroup = null;
private DNConf dnConf;
private volatile boolean heartbeatsDisabledForTests = false;
private volatile boolean cacheReportsDisabledForTests = false;
private DataStorage storage = null;
private DatanodeHttpServer httpServer = null;
@ -1055,15 +1056,27 @@ public class DataNode extends ReconfigurableBase
// used only for testing
@VisibleForTesting
void setHeartbeatsDisabledForTests(
boolean heartbeatsDisabledForTests) {
this.heartbeatsDisabledForTests = heartbeatsDisabledForTests;
}
@VisibleForTesting
boolean areHeartbeatsDisabledForTests() {
return this.heartbeatsDisabledForTests;
}
@VisibleForTesting
void setCacheReportsDisabledForTest(boolean disabled) {
this.cacheReportsDisabledForTests = disabled;
}
@VisibleForTesting
boolean areCacheReportsDisabledForTests() {
return this.cacheReportsDisabledForTests;
}
/**
* This method starts the data node with the specified conf.
*

45
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
View File

@ -44,17 +44,15 @@ import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.util.Daemon;
import org.apache.hadoop.util.DiskChecker;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.channels.FileLock;
import java.nio.channels.OverlappingFileLockException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@ -82,7 +80,6 @@ import java.util.concurrent.Future;
public class DataStorage extends Storage {
public final static String BLOCK_SUBDIR_PREFIX = "subdir";
final static String COPY_FILE_PREFIX = "dncp_";
final static String STORAGE_DIR_DETACHED = "detach";
public final static String STORAGE_DIR_RBW = "rbw";
public final static String STORAGE_DIR_FINALIZED = "finalized";
@ -614,20 +611,22 @@ public class DataStorage extends Storage {
@Override
public boolean isPreUpgradableLayout(StorageDirectory sd) throws IOException {
File oldF = new File(sd.getRoot(), "storage");
if (!oldF.exists())
if (!oldF.exists()) {
return false;
}
// check the layout version inside the storage file
// Lock and Read old storage file
RandomAccessFile oldFile = new RandomAccessFile(oldF, "rws");
FileLock oldLock = oldFile.getChannel().tryLock();
try {
try (RandomAccessFile oldFile = new RandomAccessFile(oldF, "rws");
FileLock oldLock = oldFile.getChannel().tryLock()) {
if (null == oldLock) {
LOG.error("Unable to acquire file lock on path " + oldF.toString());
throw new OverlappingFileLockException();
}
oldFile.seek(0);
int oldVersion = oldFile.readInt();
if (oldVersion < LAST_PRE_UPGRADE_LAYOUT_VERSION)
if (oldVersion < LAST_PRE_UPGRADE_LAYOUT_VERSION) {
return false;
} finally {
oldLock.release();
oldFile.close();
}
}
return true;
}
@ -1218,23 +1217,8 @@ public class DataStorage extends Storage {
return;
}
if (!from.isDirectory()) {
if (from.getName().startsWith(COPY_FILE_PREFIX)) {
FileInputStream in = new FileInputStream(from);
try {
FileOutputStream out = new FileOutputStream(to);
try {
IOUtils.copyBytes(in, out, 16*1024);
hl.linkStats.countPhysicalFileCopies++;
} finally {
out.close();
}
} finally {
in.close();
}
} else {
HardLink.createHardLink(from, to);
hl.linkStats.countSingleLinks++;
}
HardLink.createHardLink(from, to);
hl.linkStats.countSingleLinks++;
return;
}
// from is a directory
@ -1285,8 +1269,7 @@ public class DataStorage extends Storage {
String[] otherNames = from.list(new java.io.FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.startsWith(BLOCK_SUBDIR_PREFIX)
|| name.startsWith(COPY_FILE_PREFIX);
return name.startsWith(BLOCK_SUBDIR_PREFIX);
}
});
for(int i = 0; i < otherNames.length; i++)

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
View File

@ -2559,13 +2559,15 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
final String directory;
final long usedSpace; // size of space used by HDFS
final long freeSpace; // size of free space excluding reserved space
final long reservedSpace; // size of space reserved for non-HDFS and RBW
final long reservedSpace; // size of space reserved for non-HDFS
final long reservedSpaceForRBW; // size of space reserved RBW
VolumeInfo(FsVolumeImpl v, long usedSpace, long freeSpace) {
this.directory = v.toString();
this.usedSpace = usedSpace;
this.freeSpace = freeSpace;
this.reservedSpace = v.getReserved();
this.reservedSpaceForRBW = v.getReservedForRbw();
}
}
@ -2599,6 +2601,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
innerInfo.put("usedSpace", v.usedSpace);
innerInfo.put("freeSpace", v.freeSpace);
innerInfo.put("reservedSpace", v.reservedSpace);
innerInfo.put("reservedSpaceForRBW", v.reservedSpaceForRBW);
info.put(v.directory, innerInfo);
}
return info;

24
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java
View File

@ -61,6 +61,7 @@ import org.apache.hadoop.hdfs.protocol.CacheDirectiveStats;
import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolInfoProto;
@ -902,9 +903,26 @@ public final class CacheManager {
if (cachedBlock == null) {
return;
}
List<DatanodeDescriptor> datanodes = cachedBlock.getDatanodes(Type.CACHED);
for (DatanodeDescriptor datanode : datanodes) {
block.addCachedLoc(datanode);
List<DatanodeDescriptor> cachedDNs = cachedBlock.getDatanodes(Type.CACHED);
for (DatanodeDescriptor datanode : cachedDNs) {
// Filter out cached blocks that do not have a backing replica.
//
// This should not happen since it means the CacheManager thinks
// something is cached that does not exist, but it's a safety
// measure.
boolean found = false;
for (DatanodeInfo loc : block.getLocations()) {
if (loc.equals(datanode)) {
block.addCachedLoc(loc);
found = true;
break;
}
}
if (!found) {
LOG.warn("Datanode {} is not a valid cache location for block {} "
+ "because that node does not have a backing replica!",
datanode, block.getBlock().getBlockName());
}
}
}

14
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointConf.java
View File

@ -44,7 +44,13 @@ public class CheckpointConf {
/** The output dir for legacy OIV image */
private final String legacyOivImageDir;
/**
* multiplier on the checkpoint period to allow other nodes to do the checkpointing, when not the
* 'primary' checkpoint node
*/
private double quietMultiplier;
public CheckpointConf(Configuration conf) {
checkpointCheckPeriod = conf.getLong(
DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_KEY,
@ -57,6 +63,8 @@ public class CheckpointConf {
maxRetriesOnMergeError = conf.getInt(DFS_NAMENODE_CHECKPOINT_MAX_RETRIES_KEY,
DFS_NAMENODE_CHECKPOINT_MAX_RETRIES_DEFAULT);
legacyOivImageDir = conf.get(DFS_NAMENODE_LEGACY_OIV_IMAGE_DIR_KEY);
quietMultiplier = conf.getDouble(DFS_NAMENODE_CHECKPOINT_QUIET_MULTIPLIER_KEY,
DFS_NAMENODE_CHECKPOINT_QUIET_MULTIPLIER_DEFAULT);
warnForDeprecatedConfigs(conf);
}
@ -91,4 +99,8 @@ public class CheckpointConf {
public String getLegacyOivImageDir() {
return legacyOivImageDir;
}
public double getQuietPeriod() {
return this.checkpointPeriod * this.quietMultiplier;
}
}

27
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -143,7 +143,6 @@ import org.apache.hadoop.fs.CacheFlag;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsServerDefaults;
@ -3136,7 +3135,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
if (trackBlockCounts) {
if (b.isComplete()) {
numRemovedComplete++;
if (blockManager.checkMinReplication(b)) {
if (blockManager.hasMinStorage(b)) {
numRemovedSafe++;
}
}
@ -3368,7 +3367,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
curBlock = blocks[nrCompleteBlocks];
if(!curBlock.isComplete())
break;
assert blockManager.checkMinReplication(curBlock) :
assert blockManager.hasMinStorage(curBlock) :
"A COMPLETE block is not minimally replicated in " + src;
}
@ -3404,7 +3403,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
// If penultimate block doesn't exist then its minReplication is met
boolean penultimateBlockMinReplication = penultimateBlock == null ? true :
blockManager.checkMinReplication(penultimateBlock);
blockManager.hasMinStorage(penultimateBlock);
switch(lastBlockState) {
case COMPLETE:
@ -3413,7 +3412,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
case COMMITTED:
// Close file if committed blocks are minimally replicated
if(penultimateBlockMinReplication &&
blockManager.checkMinReplication(lastBlock)) {
blockManager.hasMinStorage(lastBlock)) {
finalizeINodeFileUnderConstruction(src, pendingFile,
iip.getLatestSnapshotId());
NameNode.stateChangeLog.warn("BLOCK*"
@ -3705,9 +3704,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
trimmedTargets.get(i).getStorageInfo(trimmedStorages.get(i));
if (storageInfo != null) {
if(copyTruncate) {
storageInfo.addBlock(truncatedBlock);
storageInfo.addBlock(truncatedBlock, truncatedBlock);
} else {
storageInfo.addBlock(storedBlock);
storageInfo.addBlock(storedBlock, storedBlock);
}
}
}
@ -3723,8 +3722,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
} else {
iFile.setLastBlock(storedBlock, trimmedStorageInfos);
if (closeFile) {
blockManager.markBlockReplicasAsCorrupt(storedBlock,
oldGenerationStamp, oldNumBytes, trimmedStorageInfos);
blockManager.markBlockReplicasAsCorrupt(oldBlock.getLocalBlock(),
storedBlock, oldGenerationStamp, oldNumBytes,
trimmedStorageInfos);
}
}
}
@ -6460,6 +6460,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
this.dir = dir;
}
/** @return the cache manager. */
@Override
public CacheManager getCacheManager() {
return cacheManager;
}
@ -6756,10 +6757,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
checkOperation(OperationCategory.READ);
readLock();
try {
if (rollingUpgradeInfo != null) {
boolean hasRollbackImage = this.getFSImage().hasRollbackFSImage();
rollingUpgradeInfo.setCreatedRollbackImages(hasRollbackImage);
if (!isRollingUpgrade()) {
return null;
}
Preconditions.checkNotNull(rollingUpgradeInfo);
boolean hasRollbackImage = this.getFSImage().hasRollbackFSImage();
rollingUpgradeInfo.setCreatedRollbackImages(hasRollbackImage);
return rollingUpgradeInfo;
} finally {
readUnlock();

88
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java
View File

@ -30,6 +30,7 @@ import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.commons.logging.Log;
@ -81,6 +82,9 @@ public class ImageServlet extends HttpServlet {
private static final String LATEST_FSIMAGE_VALUE = "latest";
private static final String IMAGE_FILE_TYPE = "imageFile";
private SortedSet<ImageUploadRequest> currentlyDownloadingCheckpoints = Collections
.<ImageUploadRequest> synchronizedSortedSet(new TreeSet<ImageUploadRequest>());
@Override
public void doGet(final HttpServletRequest request,
final HttpServletResponse response) throws ServletException, IOException {
@ -253,10 +257,12 @@ public class ImageServlet extends HttpServlet {
}
if (HAUtil.isHAEnabled(conf, DFSUtil.getNamenodeNameServiceId(conf))) {
Configuration otherNnConf = HAUtil.getConfForOtherNode(conf);
validRequestors.add(SecurityUtil.getServerPrincipal(otherNnConf
.get(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY),
NameNode.getAddress(otherNnConf).getHostName()));
List<Configuration> otherNnConfs = HAUtil.getConfForOtherNodes(conf);
for (Configuration otherNnConf : otherNnConfs) {
validRequestors.add(SecurityUtil.getServerPrincipal(otherNnConf
.get(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY),
NameNode.getAddress(otherNnConf).getHostName()));
}
}
for (String v : validRequestors) {
@ -420,7 +426,6 @@ public class ImageServlet extends HttpServlet {
/**
* Set the required parameters for uploading image
*
* @param httpMethod instance of method to set the parameters
* @param storage colon separated storageInfo string
* @param txid txid of the image
* @param imageFileSize size of the imagefile to be uploaded
@ -459,12 +464,37 @@ public class ImageServlet extends HttpServlet {
@Override
public Void run() throws Exception {
// if its not the active NN, then we need to notify the caller it was was the wrong
// target (regardless of the fact that we got the image)
HAServiceProtocol.HAServiceState state = NameNodeHttpServer
.getNameNodeStateFromContext(getServletContext());
if (state != HAServiceProtocol.HAServiceState.ACTIVE) {
// we need a different response type here so the client can differentiate this
// from the failure to upload due to (1) security, or (2) other checkpoints already
// present
response.sendError(HttpServletResponse.SC_EXPECTATION_FAILED,
"Nameode "+request.getLocalAddr()+" is currently not in a state which can "
+ "accept uploads of new fsimages. State: "+state);
return null;
}
final long txid = parsedParams.getTxId();
String remoteAddr = request.getRemoteAddr();
ImageUploadRequest imageRequest = new ImageUploadRequest(txid, remoteAddr);
final NameNodeFile nnf = parsedParams.getNameNodeFile();
if (!nnImage.addToCheckpointing(txid)) {
// if the node is attempting to upload an older transaction, we ignore it
SortedSet<ImageUploadRequest> larger = currentlyDownloadingCheckpoints.tailSet(imageRequest);
if (larger.size() > 0) {
response.sendError(HttpServletResponse.SC_CONFLICT,
"Another checkpointer is already in the process of uploading a" +
" checkpoint made up to transaction ID " + larger.last());
return null;
}
//make sure no one else has started uploading one
if (!currentlyDownloadingCheckpoints.add(imageRequest)) {
response.sendError(HttpServletResponse.SC_CONFLICT,
"Either current namenode is checkpointing or another"
+ " checkpointer is already in the process of "
@ -499,6 +529,10 @@ public class ImageServlet extends HttpServlet {
// remove some old ones.
nnImage.purgeOldStorage(nnf);
} finally {
// remove the request once we've processed it, or it threw an error, so we
// aren't using it either
currentlyDownloadingCheckpoints.remove(imageRequest);
stream.close();
}
} finally {
@ -555,4 +589,46 @@ public class ImageServlet extends HttpServlet {
return nnf;
}
}
private static class ImageUploadRequest implements Comparable<ImageUploadRequest> {
private final long txId;
private final String address;
public ImageUploadRequest(long txid, String remoteAddr) {
this.txId = txid;
this.address = remoteAddr;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ImageUploadRequest that = (ImageUploadRequest) o;
if (txId != that.txId) return false;
if (!address.equals(that.address)) return false;
return true;
}
@Override
public int hashCode() {
int result = (int) (txId ^ (txId >>> 32));
result = 31 * result + address.hashCode();
return result;
}
@Override public int compareTo(ImageUploadRequest other) {
return Long.compare(txId, other.txId);
}
@Override public String toString() {
return "ImageRequest{" +
"txId=" + txId +
", address='" + address + '\'' +
'}';
}
}
}

7
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
View File

@ -27,6 +27,7 @@ import javax.servlet.ServletContext;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
@ -272,4 +273,8 @@ public class NameNodeHttpServer {
ServletContext context) {
return (StartupProgress)context.getAttribute(STARTUP_PROGRESS_ATTRIBUTE_KEY);
}
}
public static HAServiceProtocol.HAServiceState getNameNodeStateFromContext(ServletContext context) {
return getNameNodeFromContext(context).getServiceState();
}
}

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java
View File

@ -81,9 +81,10 @@ public interface NameNodeMXBean {
public boolean isUpgradeFinalized();
/**
* Gets the RollingUpgrade information
* Gets the RollingUpgrade information.
*
* @return Rolling upgrade information
* @return Rolling upgrade information if an upgrade is in progress. Else
* (e.g. if there is no upgrade or the upgrade is finalized), returns null.
*/
public RollingUpgradeInfo.Bean getRollingUpgradeStatus();

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
View File

@ -647,7 +647,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
.getStorageType()));
}
if (showReplicaDetails) {
LightWeightLinkedSet<Block> blocksExcess =
LightWeightLinkedSet<BlockInfo> blocksExcess =
bm.excessReplicateMap.get(dnDesc.getDatanodeUuid());
Collection<DatanodeDescriptor> corruptReplicas =
bm.getCorruptReplicas(block.getLocalBlock());

18
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java
View File

@ -29,21 +29,23 @@ import org.apache.hadoop.security.AccessControlException;
@InterfaceAudience.Private
public interface Namesystem extends RwLock, SafeMode {
/** Is this name system running? */
public boolean isRunning();
boolean isRunning();
/** Check if the user has superuser privilege. */
public void checkSuperuserPrivilege() throws AccessControlException;
void checkSuperuserPrivilege() throws AccessControlException;
/** @return the block pool ID */
public String getBlockPoolId();
String getBlockPoolId();
public boolean isInStandbyState();
boolean isInStandbyState();
public boolean isGenStampInFuture(Block block);
boolean isGenStampInFuture(Block block);
public void adjustSafeModeBlockTotals(int deltaSafe, int deltaTotal);
void adjustSafeModeBlockTotals(int deltaSafe, int deltaTotal);
public void checkOperation(OperationCategory read) throws StandbyException;
void checkOperation(OperationCategory read) throws StandbyException;
public boolean isInSnapshot(BlockInfoUnderConstruction blockUC);
boolean isInSnapshot(BlockInfoUnderConstruction blockUC);
CacheManager getCacheManager();
}

47
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
View File

@ -70,7 +70,33 @@ import org.mortbay.jetty.EofException;
*/
@InterfaceAudience.Private
public class TransferFsImage {
public enum TransferResult{
SUCCESS(HttpServletResponse.SC_OK, false),
AUTHENTICATION_FAILURE(HttpServletResponse.SC_FORBIDDEN, true),
NOT_ACTIVE_NAMENODE_FAILURE(HttpServletResponse.SC_EXPECTATION_FAILED, false),
OLD_TRANSACTION_ID_FAILURE(HttpServletResponse.SC_CONFLICT, false),
UNEXPECTED_FAILURE(-1, true);
private final int response;
private final boolean shouldReThrowException;
private TransferResult(int response, boolean rethrow) {
this.response = response;
this.shouldReThrowException = rethrow;
}
public static TransferResult getResultForCode(int code){
TransferResult ret = UNEXPECTED_FAILURE;
for(TransferResult result:TransferResult.values()){
if(result.response == code){
return result;
}
}
return ret;
}
}
public final static String CONTENT_LENGTH = "Content-Length";
public final static String FILE_LENGTH = "File-Length";
public final static String MD5_HEADER = "X-MD5-Digest";
@ -198,9 +224,9 @@ public class TransferFsImage {
* @param txid the transaction ID of the image to be uploaded
* @throws IOException if there is an I/O error
*/
public static void uploadImageFromStorage(URL fsName, Configuration conf,
public static TransferResult uploadImageFromStorage(URL fsName, Configuration conf,
NNStorage storage, NameNodeFile nnf, long txid) throws IOException {
uploadImageFromStorage(fsName, conf, storage, nnf, txid, null);
return uploadImageFromStorage(fsName, conf, storage, nnf, txid, null);
}
/**
@ -215,7 +241,7 @@ public class TransferFsImage {
* @param canceler optional canceler to check for abort of upload
* @throws IOException if there is an I/O error or cancellation
*/
public static void uploadImageFromStorage(URL fsName, Configuration conf,
public static TransferResult uploadImageFromStorage(URL fsName, Configuration conf,
NNStorage storage, NameNodeFile nnf, long txid, Canceler canceler)
throws IOException {
URL url = new URL(fsName, ImageServlet.PATH_SPEC);
@ -223,21 +249,18 @@ public class TransferFsImage {
try {
uploadImage(url, conf, storage, nnf, txid, canceler);
} catch (HttpPutFailedException e) {
if (e.getResponseCode() == HttpServletResponse.SC_CONFLICT) {
// this is OK - this means that a previous attempt to upload
// this checkpoint succeeded even though we thought it failed.
LOG.info("Image upload with txid " + txid +
" conflicted with a previous image upload to the " +
"same NameNode. Continuing...", e);
return;
} else {
// translate the error code to a result, which is a bit more obvious in usage
TransferResult result = TransferResult.getResultForCode(e.getResponseCode());
if (result.shouldReThrowException) {
throw e;
}
return result;
}
double xferSec = Math.max(
((float) (Time.monotonicNow() - startTime)) / 1000.0, 0.001);
LOG.info("Uploaded image with txid " + txid + " to namenode at " + fsName
+ " in " + xferSec + " seconds");
return TransferResult.SUCCESS;
}
/*

92
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
View File

@ -23,8 +23,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIP
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URL;
import java.security.PrivilegedAction;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
@ -77,10 +77,8 @@ public class BootstrapStandby implements Tool, Configurable {
private static final Log LOG = LogFactory.getLog(BootstrapStandby.class);
private String nsId;
private String nnId;
private String otherNNId;
private List<RemoteNameNodeInfo> remoteNNs;
private URL otherHttpAddr;
private InetSocketAddress otherIpcAddr;
private Collection<URI> dirsToFormat;
private List<URI> editUrisToFormat;
private List<URI> sharedEditsUris;
@ -139,8 +137,8 @@ public class BootstrapStandby implements Tool, Configurable {
System.err.println("Usage: " + this.getClass().getSimpleName() +
" [-force] [-nonInteractive] [-skipSharedEditsCheck]");
}
private NamenodeProtocol createNNProtocolProxy()
private NamenodeProtocol createNNProtocolProxy(InetSocketAddress otherIpcAddr)
throws IOException {
return NameNodeProxies.createNonHAProxy(getConf(),
otherIpcAddr, NamenodeProtocol.class,
@ -149,18 +147,36 @@ public class BootstrapStandby implements Tool, Configurable {
}
private int doRun() throws IOException {
NamenodeProtocol proxy = createNNProtocolProxy();
NamespaceInfo nsInfo;
boolean isUpgradeFinalized;
try {
nsInfo = proxy.versionRequest();
isUpgradeFinalized = proxy.isUpgradeFinalized();
} catch (IOException ioe) {
LOG.fatal("Unable to fetch namespace information from active NN at " +
otherIpcAddr + ": " + ioe.getMessage());
if (LOG.isDebugEnabled()) {
LOG.debug("Full exception trace", ioe);
// find the active NN
NamenodeProtocol proxy = null;
NamespaceInfo nsInfo = null;
boolean isUpgradeFinalized = false;
RemoteNameNodeInfo proxyInfo = null;
for (int i = 0; i < remoteNNs.size(); i++) {
proxyInfo = remoteNNs.get(i);
InetSocketAddress otherIpcAddress = proxyInfo.getIpcAddress();
proxy = createNNProtocolProxy(otherIpcAddress);
try {
// Get the namespace from any active NN. If you just formatted the primary NN and are
// bootstrapping the other NNs from that layout, it will only contact the single NN.
// However, if there cluster is already running and you are adding a NN later (e.g.
// replacing a failed NN), then this will bootstrap from any node in the cluster.
nsInfo = proxy.versionRequest();
isUpgradeFinalized = proxy.isUpgradeFinalized();
break;
} catch (IOException ioe) {
LOG.warn("Unable to fetch namespace information from remote NN at " + otherIpcAddress
+ ": " + ioe.getMessage());
if (LOG.isDebugEnabled()) {
LOG.debug("Full exception trace", ioe);
}
}
}
if (nsInfo == null) {
LOG.fatal(
"Unable to fetch namespace information from any remote NN. Possible NameNodes: "
+ remoteNNs);
return ERR_CODE_FAILED_CONNECT;
}
@ -175,9 +191,9 @@ public class BootstrapStandby implements Tool, Configurable {
"=====================================================\n" +
"About to bootstrap Standby ID " + nnId + " from:\n" +
" Nameservice ID: " + nsId + "\n" +
" Other Namenode ID: " + otherNNId + "\n" +
" Other NN's HTTP address: " + otherHttpAddr + "\n" +
" Other NN's IPC address: " + otherIpcAddr + "\n" +
" Other Namenode ID: " + proxyInfo.getNameNodeID() + "\n" +
" Other NN's HTTP address: " + proxyInfo.getHttpAddress() + "\n" +
" Other NN's IPC address: " + proxyInfo.getIpcAddress() + "\n" +
" Namespace ID: " + nsInfo.getNamespaceID() + "\n" +
" Block pool ID: " + nsInfo.getBlockPoolID() + "\n" +
" Cluster ID: " + nsInfo.getClusterID() + "\n" +
@ -201,7 +217,7 @@ public class BootstrapStandby implements Tool, Configurable {
}
// download the fsimage from active namenode
int download = downloadImage(storage, proxy);
int download = downloadImage(storage, proxy, proxyInfo);
if (download != 0) {
return download;
}
@ -292,7 +308,7 @@ public class BootstrapStandby implements Tool, Configurable {
}
}
private int downloadImage(NNStorage storage, NamenodeProtocol proxy)
private int downloadImage(NNStorage storage, NamenodeProtocol proxy, RemoteNameNodeInfo proxyInfo)
throws IOException {
// Load the newly formatted image, using all of the directories
// (including shared edits)
@ -316,7 +332,7 @@ public class BootstrapStandby implements Tool, Configurable {
// Download that checkpoint into our storage directories.
MD5Hash hash = TransferFsImage.downloadImageToStorage(
otherHttpAddr, imageTxId, storage, true);
proxyInfo.getHttpAddress(), imageTxId, storage, true);
image.saveDigestAndRenameCheckpointImage(NameNodeFile.IMAGE, imageTxId,
hash);
} catch (IOException ioe) {
@ -385,18 +401,26 @@ public class BootstrapStandby implements Tool, Configurable {
throw new HadoopIllegalArgumentException(
"Shared edits storage is not enabled for this namenode.");
}
Configuration otherNode = HAUtil.getConfForOtherNode(conf);
otherNNId = HAUtil.getNameNodeId(otherNode, nsId);
otherIpcAddr = NameNode.getServiceAddress(otherNode, true);
Preconditions.checkArgument(otherIpcAddr.getPort() != 0 &&
!otherIpcAddr.getAddress().isAnyLocalAddress(),
"Could not determine valid IPC address for other NameNode (%s)" +
", got: %s", otherNNId, otherIpcAddr);
final String scheme = DFSUtil.getHttpClientScheme(conf);
otherHttpAddr = DFSUtil.getInfoServerWithDefaultHost(
otherIpcAddr.getHostName(), otherNode, scheme).toURL();
remoteNNs = RemoteNameNodeInfo.getRemoteNameNodes(conf, nsId);
// validate the configured NNs
List<RemoteNameNodeInfo> remove = new ArrayList<RemoteNameNodeInfo>(remoteNNs.size());
for (RemoteNameNodeInfo info : remoteNNs) {
InetSocketAddress address = info.getIpcAddress();
LOG.info("Found nn: " + info.getNameNodeID() + ", ipc: " + info.getIpcAddress());
if (address.getPort() == 0 || address.getAddress().isAnyLocalAddress()) {
LOG.error("Could not determine valid IPC address for other NameNode ("
+ info.getNameNodeID() + ") , got: " + address);
remove.add(info);
}
}
// remove any invalid nns
remoteNNs.removeAll(remove);
// make sure we have at least one left to read
Preconditions.checkArgument(!remoteNNs.isEmpty(), "Could not find any valid namenodes!");
dirsToFormat = FSNamesystem.getNamespaceDirs(conf);
editUrisToFormat = FSNamesystem.getNamespaceEditsDirs(

162
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
View File

@ -23,7 +23,13 @@ import java.net.InetSocketAddress;
import java.security.PrivilegedAction;
import java.security.PrivilegedExceptionAction;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.Callable;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@ -67,10 +73,10 @@ public class EditLogTailer {
private final Configuration conf;
private final FSNamesystem namesystem;
private final Iterator<RemoteNameNodeInfo> nnLookup;
private FSEditLog editLog;
private InetSocketAddress activeAddr;
private NamenodeProtocol cachedActiveProxy = null;
private RemoteNameNodeInfo currentNN;
/**
* The last transaction ID at which an edit log roll was initiated.
@ -100,7 +106,17 @@ public class EditLogTailer {
* available to be read from.
*/
private final long sleepTimeMs;
private final int nnCount;
private NamenodeProtocol cachedActiveProxy = null;
// count of the number of NNs we have attempted in the current lookup loop
private int nnLoopCount = 0;
/**
* maximum number of retries we should give each of the remote namenodes before giving up
*/
private int maxRetries;
public EditLogTailer(FSNamesystem namesystem, Configuration conf) {
this.tailerThread = new EditLogTailerThread();
this.conf = conf;
@ -111,12 +127,24 @@ public class EditLogTailer {
logRollPeriodMs = conf.getInt(DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY,
DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_DEFAULT) * 1000;
List<RemoteNameNodeInfo> nns = Collections.emptyList();
if (logRollPeriodMs >= 0) {
this.activeAddr = getActiveNodeAddress();
Preconditions.checkArgument(activeAddr.getPort() > 0,
"Active NameNode must have an IPC port configured. " +
"Got address '%s'", activeAddr);
LOG.info("Will roll logs on active node at " + activeAddr + " every " +
try {
nns = RemoteNameNodeInfo.getRemoteNameNodes(conf);
} catch (IOException e) {
throw new IllegalArgumentException("Remote NameNodes not correctly configured!", e);
}
for (RemoteNameNodeInfo info : nns) {
// overwrite the socket address, if we need to
InetSocketAddress ipc = NameNode.getServiceAddress(info.getConfiguration(), true);
// sanity check the ipc address
Preconditions.checkArgument(ipc.getPort() > 0,
"Active NameNode must have an IPC port configured. " + "Got address '%s'", ipc);
info.setIpcAddress(ipc);
}
LOG.info("Will roll logs on active node every " +
(logRollPeriodMs / 1000) + " seconds.");
} else {
LOG.info("Not going to trigger log rolls on active node because " +
@ -125,29 +153,24 @@ public class EditLogTailer {
sleepTimeMs = conf.getInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY,
DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_DEFAULT) * 1000;
maxRetries = conf.getInt(DFSConfigKeys.DFS_HA_TAILEDITS_ALL_NAMESNODES_RETRY_KEY,
DFSConfigKeys.DFS_HA_TAILEDITS_ALL_NAMESNODES_RETRY_DEFAULT);
if (maxRetries <= 0) {
LOG.error("Specified a non-positive number of retries for the number of retries for the " +
"namenode connection when manipulating the edit log (" +
DFSConfigKeys.DFS_HA_TAILEDITS_ALL_NAMESNODES_RETRY_KEY + "), setting to default: " +
DFSConfigKeys.DFS_HA_TAILEDITS_ALL_NAMESNODES_RETRY_DEFAULT);
maxRetries = DFSConfigKeys.DFS_HA_TAILEDITS_ALL_NAMESNODES_RETRY_DEFAULT;
}
nnCount = nns.size();
// setup the iterator to endlessly loop the nns
this.nnLookup = Iterators.cycle(nns);
LOG.debug("logRollPeriodMs=" + logRollPeriodMs +
" sleepTime=" + sleepTimeMs);
}
private InetSocketAddress getActiveNodeAddress() {
Configuration activeConf = HAUtil.getConfForOtherNode(conf);
return NameNode.getServiceAddress(activeConf, true);
}
private NamenodeProtocol getActiveNodeProxy() throws IOException {
if (cachedActiveProxy == null) {
int rpcTimeout = conf.getInt(
DFSConfigKeys.DFS_HA_LOGROLL_RPC_TIMEOUT_KEY,
DFSConfigKeys.DFS_HA_LOGROLL_RPC_TIMEOUT_DEFAULT);
NamenodeProtocolPB proxy = RPC.waitForProxy(NamenodeProtocolPB.class,
RPC.getProtocolVersion(NamenodeProtocolPB.class), activeAddr, conf,
rpcTimeout, Long.MAX_VALUE);
cachedActiveProxy = new NamenodeProtocolTranslatorPB(proxy);
}
assert cachedActiveProxy != null;
return cachedActiveProxy;
}
public void start() {
tailerThread.start();
@ -270,9 +293,15 @@ public class EditLogTailer {
* Trigger the active node to roll its logs.
*/
private void triggerActiveLogRoll() {
LOG.info("Triggering log roll on remote NameNode " + activeAddr);
LOG.info("Triggering log roll on remote NameNode");
try {
getActiveNodeProxy().rollEditLog();
new MultipleNameNodeProxy<Void>() {
@Override
protected Void doWork() throws IOException {
cachedActiveProxy.rollEditLog();
return null;
}
}.call();
lastRollTriggerTxId = lastLoadedTxnId;
} catch (IOException ioe) {
if (ioe instanceof RemoteException) {
@ -362,5 +391,76 @@ public class EditLogTailer {
}
}
}
/**
* Manage the 'active namenode proxy'. This cannot just be the a single proxy since we could
* failover across a number of NameNodes, rather than just between an active and a standby.
* <p>
* We - lazily - get a proxy to one of the configured namenodes and attempt to make the request
* against it. If it doesn't succeed, either because the proxy failed to be created or the request
* failed, we try the next NN in the list. We try this up to the configuration maximum number of
* retries before throwing up our hands. A working proxy is retained across attempts since we
* expect the active NameNode to switch rarely.
* <p>
* This mechanism is <b>very bad</b> for cases where we care about being <i>fast</i>; it just
* blindly goes and tries namenodes.
*/
private abstract class MultipleNameNodeProxy<T> implements Callable<T> {
}
/**
* Do the actual work to the remote namenode via the {@link #cachedActiveProxy}.
* @return the result of the work, if there is one
* @throws IOException if the actions done to the proxy throw an exception.
*/
protected abstract T doWork() throws IOException;
public T call() throws IOException {
while ((cachedActiveProxy = getActiveNodeProxy()) != null) {
try {
T ret = doWork();
// reset the loop count on success
nnLoopCount = 0;
return ret;
} catch (RemoteException e) {
Throwable cause = e.unwrapRemoteException(StandbyException.class);
// if its not a standby exception, then we need to re-throw it, something bad has happened
if (cause == e) {
throw e;
} else {
// it is a standby exception, so we try the other NN
LOG.warn("Failed to reach remote node: " + currentNN
+ ", retrying with remaining remote NNs");
cachedActiveProxy = null;
// this NN isn't responding to requests, try the next one
nnLoopCount++;
}
}
}
throw new IOException("Cannot find any valid remote NN to service request!");
}
private NamenodeProtocol getActiveNodeProxy() throws IOException {
if (cachedActiveProxy == null) {
while (true) {
// if we have reached the max loop count, quit by returning null
if ((nnLoopCount / nnCount) >= maxRetries) {
return null;
}
currentNN = nnLookup.next();
try {
NamenodeProtocolPB proxy = RPC.waitForProxy(NamenodeProtocolPB.class,
RPC.getProtocolVersion(NamenodeProtocolPB.class), currentNN.getIpcAddress(), conf);
cachedActiveProxy = new NamenodeProtocolTranslatorPB(proxy);
break;
} catch (IOException e) {
LOG.info("Failed to reach " + currentNN, e);
// couldn't even reach this NN, try the next one
nnLoopCount++;
}
}
}
assert cachedActiveProxy != null;
return cachedActiveProxy;
}
}
}

133
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RemoteNameNodeInfo.java Normal file
View File

@ -0,0 +1,133 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode.ha;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import com.google.common.base.Objects;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HAUtil;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import com.google.common.base.Preconditions;
/**
* Information about a single remote NameNode
*/
public class RemoteNameNodeInfo {
public static List<RemoteNameNodeInfo> getRemoteNameNodes(Configuration conf) throws IOException {
String nsId = DFSUtil.getNamenodeNameServiceId(conf);
return getRemoteNameNodes(conf, nsId);
}
public static List<RemoteNameNodeInfo> getRemoteNameNodes(Configuration conf, String nsId)
throws IOException {
// there is only a single NN configured (and no federation) so we don't have any more NNs
if (nsId == null) {
return Collections.emptyList();
}
List<Configuration> otherNodes = HAUtil.getConfForOtherNodes(conf);
List<RemoteNameNodeInfo> nns = new ArrayList<RemoteNameNodeInfo>();
for (Configuration otherNode : otherNodes) {
String otherNNId = HAUtil.getNameNodeId(otherNode, nsId);
// don't do any validation here as in some cases, it can be overwritten later
InetSocketAddress otherIpcAddr = NameNode.getServiceAddress(otherNode, true);
final String scheme = DFSUtil.getHttpClientScheme(conf);
URL otherHttpAddr = DFSUtil.getInfoServerWithDefaultHost(otherIpcAddr.getHostName(),
otherNode, scheme).toURL();
nns.add(new RemoteNameNodeInfo(otherNode, otherNNId, otherIpcAddr, otherHttpAddr));
}
return nns;
}
private final Configuration conf;
private final String nnId;
private InetSocketAddress ipcAddress;
private final URL httpAddress;
private RemoteNameNodeInfo(Configuration conf, String nnId, InetSocketAddress ipcAddress,
URL httpAddress) {
this.conf = conf;
this.nnId = nnId;
this.ipcAddress = ipcAddress;
this.httpAddress = httpAddress;
}
public InetSocketAddress getIpcAddress() {
return this.ipcAddress;
}
public String getNameNodeID() {
return this.nnId;
}
public URL getHttpAddress() {
return this.httpAddress;
}
public Configuration getConfiguration() {
return this.conf;
}
public void setIpcAddress(InetSocketAddress ipc) {
this.ipcAddress = ipc;
}
@Override
public String toString() {
return "RemoteNameNodeInfo [nnId=" + nnId + ", ipcAddress=" + ipcAddress
+ ", httpAddress=" + httpAddress + "]";
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
RemoteNameNodeInfo that = (RemoteNameNodeInfo) o;
if (!nnId.equals(that.nnId)) return false;
if (!ipcAddress.equals(that.ipcAddress)) return false;
// convert to the standard strings since URL.equals does address resolution, which is a
// blocking call and a a FindBugs issue.
String httpString = httpAddress.toString();
String thatHttpString = that.httpAddress.toString();
return httpString.equals(thatHttpString);
}
@Override
public int hashCode() {
int result = nnId.hashCode();
result = 31 * result + ipcAddress.hashCode();
// toString rather than hashCode b/c Url.hashCode is a blocking call.
result = 31 * result + httpAddress.toString().hashCode();
return result;
}
}

182
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java
View File

@ -23,12 +23,10 @@ import java.io.IOException;
import java.net.URI;
import java.net.URL;
import java.security.PrivilegedAction;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -45,6 +43,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.SaveNamespaceCancelledException;
import org.apache.hadoop.hdfs.server.namenode.TransferFsImage;
import org.apache.hadoop.hdfs.util.Canceler;
import org.apache.hadoop.io.MultipleIOException;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
@ -68,12 +67,13 @@ public class StandbyCheckpointer {
private long lastCheckpointTime;
private final CheckpointerThread thread;
private final ThreadFactory uploadThreadFactory;
private URL activeNNAddress;
private List<URL> activeNNAddresses;
private URL myNNAddress;
private final Object cancelLock = new Object();
private Canceler canceler;
private boolean isPrimaryCheckPointer = true;
// Keep track of how many checkpoints were canceled.
// This is for use in tests.
private static int canceledCount = 0;
@ -100,14 +100,21 @@ public class StandbyCheckpointer {
myNNAddress = getHttpAddress(conf);
// Look up the active node's address
Configuration confForActive = HAUtil.getConfForOtherNode(conf);
activeNNAddress = getHttpAddress(confForActive);
List<Configuration> confForActive = HAUtil.getConfForOtherNodes(conf);
activeNNAddresses = new ArrayList<URL>(confForActive.size());
for (Configuration activeConf : confForActive) {
URL activeNNAddress = getHttpAddress(activeConf);
// sanity check each possible active NN
Preconditions.checkArgument(checkAddress(activeNNAddress),
"Bad address for active NN: %s", activeNNAddress);
activeNNAddresses.add(activeNNAddress);
}
// Sanity-check.
Preconditions.checkArgument(checkAddress(activeNNAddress),
"Bad address for active NN: %s", activeNNAddress);
Preconditions.checkArgument(checkAddress(myNNAddress),
"Bad address for standby NN: %s", myNNAddress);
Preconditions.checkArgument(checkAddress(myNNAddress), "Bad address for standby NN: %s",
myNNAddress);
}
private URL getHttpAddress(Configuration conf) throws IOException {
@ -127,7 +134,7 @@ public class StandbyCheckpointer {
public void start() {
LOG.info("Starting standby checkpoint thread...\n" +
"Checkpointing active NN at " + activeNNAddress + "\n" +
"Checkpointing active NN to possible NNs: " + activeNNAddresses + "\n" +
"Serving checkpoints at " + myNNAddress);
thread.start();
}
@ -148,11 +155,10 @@ public class StandbyCheckpointer {
thread.interrupt();
}
private void doCheckpoint() throws InterruptedException, IOException {
private void doCheckpoint(boolean sendCheckpoint) throws InterruptedException, IOException {
assert canceler != null;
final long txid;
final NameNodeFile imageType;
// Acquire cpLock to make sure no one is modifying the name system.
// It does not need the full namesystem write lock, since the only thing
// that modifies namesystem on standby node is edit log replaying.
@ -161,9 +167,9 @@ public class StandbyCheckpointer {
assert namesystem.getEditLog().isOpenForRead() :
"Standby Checkpointer should only attempt a checkpoint when " +
"NN is in standby mode, but the edit logs are in an unexpected state";
FSImage img = namesystem.getFSImage();
long prevCheckpointTxId = img.getStorage().getMostRecentCheckpointTxId();
long thisCheckpointTxId = img.getLastAppliedOrWrittenTxId();
assert thisCheckpointTxId >= prevCheckpointTxId;
@ -185,7 +191,7 @@ public class StandbyCheckpointer {
img.saveNamespace(namesystem, imageType, canceler);
txid = img.getStorage().getMostRecentCheckpointTxId();
assert txid == thisCheckpointTxId : "expected to save checkpoint at txid=" +
thisCheckpointTxId + " but instead saved at txid=" + txid;
thisCheckpointTxId + " but instead saved at txid=" + txid;
// Save the legacy OIV image, if the output dir is defined.
String outputDir = checkpointConf.getLegacyOivImageDir();
@ -195,31 +201,85 @@ public class StandbyCheckpointer {
} finally {
namesystem.cpUnlock();
}
//early exit if we shouldn't actually send the checkpoint to the ANN
if(!sendCheckpoint){
return;
}
// Upload the saved checkpoint back to the active
// Do this in a separate thread to avoid blocking transition to active
// Do this in a separate thread to avoid blocking transition to active, but don't allow more
// than the expected number of tasks to run or queue up
// See HDFS-4816
ExecutorService executor =
Executors.newSingleThreadExecutor(uploadThreadFactory);
Future<Void> upload = executor.submit(new Callable<Void>() {
@Override
public Void call() throws IOException {
TransferFsImage.uploadImageFromStorage(activeNNAddress, conf,
namesystem.getFSImage().getStorage(), imageType, txid, canceler);
return null;
ExecutorService executor = new ThreadPoolExecutor(0, activeNNAddresses.size(), 100,
TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(activeNNAddresses.size()),
uploadThreadFactory);
// for right now, just match the upload to the nn address by convention. There is no need to
// directly tie them together by adding a pair class.
List<Future<TransferFsImage.TransferResult>> uploads =
new ArrayList<Future<TransferFsImage.TransferResult>>();
for (final URL activeNNAddress : activeNNAddresses) {
Future<TransferFsImage.TransferResult> upload =
executor.submit(new Callable<TransferFsImage.TransferResult>() {
@Override
public TransferFsImage.TransferResult call() throws IOException {
return TransferFsImage.uploadImageFromStorage(activeNNAddress, conf, namesystem
.getFSImage().getStorage(), imageType, txid, canceler);
}
});
uploads.add(upload);
}
InterruptedException ie = null;
IOException ioe= null;
int i = 0;
boolean success = false;
for (; i < uploads.size(); i++) {
Future<TransferFsImage.TransferResult> upload = uploads.get(i);
try {
// TODO should there be some smarts here about retries nodes that are not the active NN?
if (upload.get() == TransferFsImage.TransferResult.SUCCESS) {
success = true;
//avoid getting the rest of the results - we don't care since we had a successful upload
break;
}
} catch (ExecutionException e) {
ioe = new IOException("Exception during image upload: " + e.getMessage(),
e.getCause());
break;
} catch (InterruptedException e) {
ie = e;
break;
}
}
// we are primary if we successfully updated the ANN
this.isPrimaryCheckPointer = success;
// cleaner than copying code for multiple catch statements and better than catching all
// exceptions, so we just handle the ones we expect.
if (ie != null || ioe != null) {
// cancel the rest of the tasks, and close the pool
for (; i < uploads.size(); i++) {
Future<TransferFsImage.TransferResult> upload = uploads.get(i);
// The background thread may be blocked waiting in the throttler, so
// interrupt it.
upload.cancel(true);
}
// shutdown so we interrupt anything running and don't start anything new
executor.shutdownNow();
// this is a good bit longer than the thread timeout, just to make sure all the threads
// that are not doing any work also stop
executor.awaitTermination(500, TimeUnit.MILLISECONDS);
// re-throw the exception we got, since one of these two must be non-null
if (ie != null) {
throw ie;
} else if (ioe != null) {
throw ioe;
}
});
executor.shutdown();
try {
upload.get();
} catch (InterruptedException e) {
// The background thread may be blocked waiting in the throttler, so
// interrupt it.
upload.cancel(true);
throw e;
} catch (ExecutionException e) {
throw new IOException("Exception during image upload: " + e.getMessage(),
e.getCause());
}
}
@ -322,8 +382,10 @@ public class StandbyCheckpointer {
final long now = monotonicNow();
final long uncheckpointed = countUncheckpointedTxns();
final long secsSinceLast = (now - lastCheckpointTime) / 1000;
// if we need a rollback checkpoint, always attempt to checkpoint
boolean needCheckpoint = needRollbackCheckpoint;
if (needCheckpoint) {
LOG.info("Triggering a rollback fsimage for rolling upgrade.");
} else if (uncheckpointed >= checkpointConf.getTxnCount()) {
@ -338,19 +400,23 @@ public class StandbyCheckpointer {
"exceeds the configured interval " + checkpointConf.getPeriod());
needCheckpoint = true;
}
synchronized (cancelLock) {
if (now < preventCheckpointsUntil) {
LOG.info("But skipping this checkpoint since we are about to failover!");
canceledCount++;
continue;
}
assert canceler == null;
canceler = new Canceler();
}
if (needCheckpoint) {
doCheckpoint();
synchronized (cancelLock) {
if (now < preventCheckpointsUntil) {
LOG.info("But skipping this checkpoint since we are about to failover!");
canceledCount++;
continue;
}
assert canceler == null;
canceler = new Canceler();
}
// on all nodes, we build the checkpoint. However, we only ship the checkpoint if have a
// rollback request, are the checkpointer, are outside the quiet period.
boolean sendRequest = isPrimaryCheckPointer || secsSinceLast >= checkpointConf.getQuietPeriod();
doCheckpoint(sendRequest);
// reset needRollbackCheckpoint to false only when we finish a ckpt
// for rollback image
if (needRollbackCheckpoint
@ -379,7 +445,7 @@ public class StandbyCheckpointer {
}
@VisibleForTesting
URL getActiveNNAddress() {
return activeNNAddress;
List<URL> getActiveNNAddresses() {
return activeNNAddresses;
}
}

13
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java
View File

@ -25,6 +25,8 @@ import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
@ -261,4 +263,15 @@ public class DFSZKFailoverController extends ZKFailoverController {
return isThreadDumpCaptured;
}
@Override
public List<HAServiceTarget> getAllOtherNodes() {
String nsId = DFSUtil.getNamenodeNameServiceId(conf);
List<String> otherNn = HAUtil.getNameNodeIdOfOtherNodes(conf, nsId);
List<HAServiceTarget> targets = new ArrayList<HAServiceTarget>(otherNn.size());
for (String nnId : otherNn) {
targets.add(new NNHAServiceTarget(conf, nsId, nnId));
}
return targets;
}
}

57
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java
View File

@ -22,6 +22,7 @@ import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_LENGTH;
import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_TYPE;
import static io.netty.handler.codec.http.HttpHeaderValues.CLOSE;
import static io.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST;
import static io.netty.handler.codec.http.HttpResponseStatus.FORBIDDEN;
import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR;
import static io.netty.handler.codec.http.HttpResponseStatus.METHOD_NOT_ALLOWED;
import static io.netty.handler.codec.http.HttpResponseStatus.NOT_FOUND;
@ -74,10 +75,10 @@ class FSImageHandler extends SimpleChannelInboundHandler<HttpRequest> {
@Override
public void channelRead0(ChannelHandlerContext ctx, HttpRequest request)
throws Exception {
throws Exception {
if (request.method() != HttpMethod.GET) {
DefaultHttpResponse resp = new DefaultHttpResponse(HTTP_1_1,
METHOD_NOT_ALLOWED);
METHOD_NOT_ALLOWED);
resp.headers().set(CONNECTION, CLOSE);
ctx.write(resp).addListener(ChannelFutureListener.CLOSE);
return;
@ -89,25 +90,33 @@ class FSImageHandler extends SimpleChannelInboundHandler<HttpRequest> {
final String content;
String path = getPath(decoder);
switch (op) {
case "GETFILESTATUS":
content = image.getFileStatus(path);
break;
case "LISTSTATUS":
content = image.listStatus(path);
break;
case "GETACLSTATUS":
content = image.getAclStatus(path);
break;
default:
throw new IllegalArgumentException(
"Invalid value for webhdfs parameter" + " \"op\"");
case "GETFILESTATUS":
content = image.getFileStatus(path);
break;
case "LISTSTATUS":
content = image.listStatus(path);
break;
case "GETACLSTATUS":
content = image.getAclStatus(path);
break;
case "GETXATTRS":
List<String> names = getXattrNames(decoder);
String encoder = getEncoder(decoder);
content = image.getXAttrs(path, names, encoder);
break;
case "LISTXATTRS":
content = image.listXAttrs(path);
break;
default:
throw new IllegalArgumentException("Invalid value for webhdfs parameter"
+ " \"op\"");
}
LOG.info("op=" + op + " target=" + path);
DefaultFullHttpResponse resp = new DefaultFullHttpResponse(
HTTP_1_1, HttpResponseStatus.OK,
Unpooled.wrappedBuffer(content.getBytes(Charsets.UTF_8)));
DefaultFullHttpResponse resp = new DefaultFullHttpResponse(HTTP_1_1,
HttpResponseStatus.OK, Unpooled.wrappedBuffer(content
.getBytes(Charsets.UTF_8)));
resp.headers().set(CONTENT_TYPE, APPLICATION_JSON_UTF8);
resp.headers().set(CONTENT_LENGTH, resp.content().readableBytes());
resp.headers().set(CONNECTION, CLOSE);
@ -134,8 +143,9 @@ class FSImageHandler extends SimpleChannelInboundHandler<HttpRequest> {
resp.setStatus(BAD_REQUEST);
} else if (e instanceof FileNotFoundException) {
resp.setStatus(NOT_FOUND);
} else if (e instanceof IOException) {
resp.setStatus(FORBIDDEN);
}
resp.headers().set(CONTENT_LENGTH, resp.content().readableBytes());
resp.headers().set(CONNECTION, CLOSE);
ctx.write(resp).addListener(ChannelFutureListener.CLOSE);
@ -147,6 +157,17 @@ class FSImageHandler extends SimpleChannelInboundHandler<HttpRequest> {
? StringUtils.toUpperCase(parameters.get("op").get(0)) : null;
}
private static List<String> getXattrNames(QueryStringDecoder decoder) {
Map<String, List<String>> parameters = decoder.parameters();
return parameters.get("xattr.name");
}
private static String getEncoder(QueryStringDecoder decoder) {
Map<String, List<String>> parameters = decoder.parameters();
return parameters.containsKey("encoding") ? parameters.get("encoding").get(
0) : null;
}
private static String getPath(QueryStringDecoder decoder)
throws FileNotFoundException {
String path = decoder.path();

75
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
View File

@ -18,7 +18,6 @@
package org.apache.hadoop.hdfs.tools.offlineImageViewer;
import java.io.BufferedInputStream;
import java.io.EOFException;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
@ -38,10 +37,12 @@ import com.google.protobuf.InvalidProtocolBufferException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf;
@ -49,6 +50,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSImageUtil;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto;
import org.apache.hadoop.hdfs.server.namenode.INodeId;
import org.apache.hadoop.hdfs.web.JsonUtil;
import org.apache.hadoop.hdfs.web.resources.XAttrEncodingParam;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.LimitInputStream;
import org.codehaus.jackson.map.ObjectMapper;
@ -308,6 +310,77 @@ class FSImageLoader {
return list;
}
/**
* Return the JSON formatted XAttrNames of the specified file.
*
* @param path
* a path specifies a file
* @return JSON formatted XAttrNames
* @throws IOException
* if failed to serialize fileStatus to JSON.
*/
String listXAttrs(String path) throws IOException {
return JsonUtil.toJsonString(getXAttrList(path));
}
/**
* Return the JSON formatted XAttrs of the specified file.
*
* @param path
* a path specifies a file
* @return JSON formatted XAttrs
* @throws IOException
* if failed to serialize fileStatus to JSON.
*/
String getXAttrs(String path, List<String> names, String encoder)
throws IOException {
List<XAttr> xAttrs = getXAttrList(path);
List<XAttr> filtered;
if (names == null || names.size() == 0) {
filtered = xAttrs;
} else {
filtered = Lists.newArrayListWithCapacity(names.size());
for (String name : names) {
XAttr search = XAttrHelper.buildXAttr(name);
boolean found = false;
for (XAttr aXAttr : xAttrs) {
if (aXAttr.getNameSpace() == search.getNameSpace()
&& aXAttr.getName().equals(search.getName())) {
filtered.add(aXAttr);
found = true;
break;
}
}
if (!found) {
throw new IOException(
"At least one of the attributes provided was not found.");
}
}
}
return JsonUtil.toJsonString(filtered,
new XAttrEncodingParam(encoder).getEncoding());
}
private List<XAttr> getXAttrList(String path) throws IOException {
long id = lookup(path);
FsImageProto.INodeSection.INode inode = fromINodeId(id);
switch (inode.getType()) {
case FILE:
return FSImageFormatPBINode.Loader.loadXAttrs(
inode.getFile().getXAttrs(), stringTable);
case DIRECTORY:
return FSImageFormatPBINode.Loader.loadXAttrs(inode.getDirectory()
.getXAttrs(), stringTable);
default:
return null;
}
}
/**
* Return the JSON formatted ACL status of the specified file.
* @param path a path specifies a file

4
hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_open.c
View File

@ -24,9 +24,6 @@
#include <stdio.h>
#include <stdlib.h>
static int get_hdfs_open_flags_from_info(hdfsFS fs, const char *path,
int flags, int *outflags, const hdfsFileInfo *info);
/**
* Given a set of FUSE flags, determine the libhdfs flags we need.
*
@ -47,7 +44,6 @@ static int get_hdfs_open_flags_from_info(hdfsFS fs, const char *path,
*/
static int64_t get_hdfs_open_flags(hdfsFS fs, const char *path, int flags)
{
int hasContent;
int64_t ret;
hdfsFileInfo *info;

20
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -913,6 +913,18 @@
</description>
</property>
<property>
<name>dfs.namenode.checkpoint.check.quiet-multiplier</name>
<value>1.5</value>
<description>
Used to calculate the amount of time between retries when in the 'quiet' period
for creating checkpoints (active namenode already has an up-to-date image from another
checkpointer), so we wait a multiplier of the dfs.namenode.checkpoint.check.period before
retrying the checkpoint because another node likely is already managing the checkpoints,
allowing us to save bandwidth to transfer checkpoints that don't need to be used.
</description>
</property>
<property>
<name>dfs.namenode.num.checkpoints.retained</name>
<value>2</value>
@ -1287,6 +1299,14 @@
</description>
</property>
<property>
<name>dfs.ha.tail-edits.namenode-retries</name>
<value>3</value>
<description>
Number of retries to use when contacting the namenode when tailing the log.
</description>
</property>
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>false</value>

2
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
View File

@ -98,6 +98,8 @@ The Web processor now supports the following operations:
* [LISTSTATUS](./WebHDFS.html#List_a_Directory)
* [GETFILESTATUS](./WebHDFS.html#Status_of_a_FileDirectory)
* [GETACLSTATUS](./WebHDFS.html#Get_ACL_Status)
* [GETXATTRS](./WebHDFS.html#Get_an_XAttr)
* [LISTXATTRS](./WebHDFS.html#List_all_XAttrs)
### XML Processor

25
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
View File

@ -79,6 +79,7 @@ import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileSystem.Statistics;
import org.apache.hadoop.fs.FsShell;
@ -526,6 +527,30 @@ public class DFSTestUtil {
}
}
public static void waitForReplication(final DistributedFileSystem dfs,
final Path file, final short replication, int waitForMillis)
throws TimeoutException, InterruptedException {
GenericTestUtils.waitFor(new Supplier<Boolean>() {
@Override
public Boolean get() {
try {
FileStatus stat = dfs.getFileStatus(file);
BlockLocation[] locs = dfs.getFileBlockLocations(stat, 0, stat
.getLen());
for (BlockLocation loc : locs) {
if (replication != loc.getHosts().length) {
return false;
}
}
return true;
} catch (IOException e) {
LOG.info("getFileStatus on path " + file + " failed!", e);
return false;
}
}
}, 100, waitForMillis);
}
/**
* Keep accessing the given file until the namenode reports that the
* given block in the file contains the given number of corrupt replicas.

464
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
View File

@ -62,6 +62,8 @@ import java.util.Map;
import java.util.Random;
import java.util.Set;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@ -445,7 +447,7 @@ public class MiniDFSCluster {
final int numNameNodes = builder.nnTopology.countNameNodes();
LOG.info("starting cluster: numNameNodes=" + numNameNodes
+ ", numDataNodes=" + builder.numDataNodes);
nameNodes = new NameNodeInfo[numNameNodes];
this.storagesPerDatanode = builder.storagesPerDatanode;
// Duplicate the storageType setting for each DN.
@ -515,7 +517,7 @@ public class MiniDFSCluster {
}
private Configuration conf;
private NameNodeInfo[] nameNodes;
private Multimap<String, NameNodeInfo> namenodes = ArrayListMultimap.create();
protected int numDataNodes;
protected final List<DataNodeProperties> dataNodes =
new ArrayList<DataNodeProperties>();
@ -539,10 +541,10 @@ public class MiniDFSCluster {
* Stores the information related to a namenode in the cluster
*/
public static class NameNodeInfo {
final NameNode nameNode;
final Configuration conf;
final String nameserviceId;
final String nnId;
public NameNode nameNode;
Configuration conf;
String nameserviceId;
String nnId;
StartupOption startOpt;
NameNodeInfo(NameNode nn, String nameserviceId, String nnId,
StartupOption startOpt, Configuration conf) {
@ -563,7 +565,6 @@ public class MiniDFSCluster {
* without a name node (ie when the name node is started elsewhere).
*/
public MiniDFSCluster() {
nameNodes = new NameNodeInfo[0]; // No namenode in the cluster
storagesPerDatanode = DEFAULT_STORAGES_PER_DATANODE;
synchronized (MiniDFSCluster.class) {
instanceId = instanceCount++;
@ -740,7 +741,6 @@ public class MiniDFSCluster {
StartupOption operation,
String[] racks, String hosts[],
long[] simulatedCapacities) throws IOException {
this.nameNodes = new NameNodeInfo[1]; // Single namenode in the cluster
this.storagesPerDatanode = DEFAULT_STORAGES_PER_DATANODE;
initMiniDFSCluster(conf, numDataNodes, null, format,
manageNameDfsDirs, true, manageDataDfsDirs, manageDataDfsDirs,
@ -814,7 +814,7 @@ public class MiniDFSCluster {
createNameNodesAndSetConf(
nnTopology, manageNameDfsDirs, manageNameDfsSharedDirs,
enableManagedDfsDirsRedundancy,
format, startOpt, clusterId, conf);
format, startOpt, clusterId);
} catch (IOException ioe) {
LOG.error("IOE creating namenodes. Permissions dump:\n" +
createPermissionsDiagnosisString(data_dir), ioe);
@ -871,7 +871,127 @@ public class MiniDFSCluster {
private void createNameNodesAndSetConf(MiniDFSNNTopology nnTopology,
boolean manageNameDfsDirs, boolean manageNameDfsSharedDirs,
boolean enableManagedDfsDirsRedundancy, boolean format,
StartupOption operation, String clusterId) throws IOException {
// do the basic namenode configuration
configureNameNodes(nnTopology, federation, conf);
int nnCounter = 0;
int nsCounter = 0;
// configure each NS independently
for (MiniDFSNNTopology.NSConf nameservice : nnTopology.getNameservices()) {
configureNameService(nameservice, nsCounter++, manageNameDfsSharedDirs,
manageNameDfsDirs, enableManagedDfsDirsRedundancy,
format, operation, clusterId, nnCounter);
nnCounter += nameservice.getNNs().size();
}
}
/**
* Do the rest of the NN configuration for things like shared edits,
* as well as directory formatting, etc. for a single nameservice
* @param nnCounter the count of the number of namenodes already configured/started. Also,
* acts as the <i>index</i> to the next NN to start (since indicies start at 0).
* @throws IOException
*/
private void configureNameService(MiniDFSNNTopology.NSConf nameservice, int nsCounter,
boolean manageNameDfsSharedDirs, boolean manageNameDfsDirs, boolean
enableManagedDfsDirsRedundancy, boolean format,
StartupOption operation, String clusterId,
final int nnCounter) throws IOException{
String nsId = nameservice.getId();
String lastDefaultFileSystem = null;
// If HA is enabled on this nameservice, enumerate all the namenodes
// in the configuration. Also need to set a shared edits dir
int numNNs = nameservice.getNNs().size();
if (numNNs > 1 && manageNameDfsSharedDirs) {
URI sharedEditsUri = getSharedEditsDir(nnCounter, nnCounter + numNNs - 1);
conf.set(DFS_NAMENODE_SHARED_EDITS_DIR_KEY, sharedEditsUri.toString());
// Clean out the shared edits dir completely, including all subdirectories.
FileUtil.fullyDelete(new File(sharedEditsUri));
}
// Now format first NN and copy the storage directory from that node to the others.
int nnIndex = nnCounter;
Collection<URI> prevNNDirs = null;
for (NNConf nn : nameservice.getNNs()) {
initNameNodeConf(conf, nsId, nsCounter, nn.getNnId(), manageNameDfsDirs,
manageNameDfsDirs, nnIndex);
Collection<URI> namespaceDirs = FSNamesystem.getNamespaceDirs(conf);
if (format) {
// delete the existing namespaces
for (URI nameDirUri : namespaceDirs) {
File nameDir = new File(nameDirUri);
if (nameDir.exists() && !FileUtil.fullyDelete(nameDir)) {
throw new IOException("Could not fully delete " + nameDir);
}
}
// delete the checkpoint directories, if they exist
Collection<URI> checkpointDirs = Util.stringCollectionAsURIs(conf
.getTrimmedStringCollection(DFS_NAMENODE_CHECKPOINT_DIR_KEY));
for (URI checkpointDirUri : checkpointDirs) {
File checkpointDir = new File(checkpointDirUri);
if (checkpointDir.exists() && !FileUtil.fullyDelete(checkpointDir)) {
throw new IOException("Could not fully delete " + checkpointDir);
}
}
}
boolean formatThisOne = format;
// if we are looking at not the first NN
if (nnIndex++ > nnCounter && format) {
// Don't format the second, third, etc NN in an HA setup - that
// would result in it having a different clusterID,
// block pool ID, etc. Instead, copy the name dirs
// from the previous one.
formatThisOne = false;
assert (null != prevNNDirs);
copyNameDirs(prevNNDirs, namespaceDirs, conf);
}
if (formatThisOne) {
// Allow overriding clusterID for specific NNs to test
// misconfiguration.
if (nn.getClusterId() == null) {
StartupOption.FORMAT.setClusterId(clusterId);
} else {
StartupOption.FORMAT.setClusterId(nn.getClusterId());
}
DFSTestUtil.formatNameNode(conf);
}
prevNNDirs = namespaceDirs;
}
// create all the namenodes in the namespace
nnIndex = nnCounter;
for (NNConf nn : nameservice.getNNs()) {
initNameNodeConf(conf, nsId, nsCounter, nn.getNnId(), manageNameDfsDirs,
enableManagedDfsDirsRedundancy, nnIndex++);
NameNodeInfo info = createNameNode(conf, false, operation,
clusterId, nsId, nn.getNnId());
// Record the last namenode uri
if (info != null && info.conf != null) {
lastDefaultFileSystem =
info.conf.get(FS_DEFAULT_NAME_KEY);
}
}
if (!federation && lastDefaultFileSystem != null) {
// Set the default file system to the actual bind address of NN.
conf.set(FS_DEFAULT_NAME_KEY, lastDefaultFileSystem);
}
}
/**
* Do the basic NN configuration for the topology. Does not configure things like the shared
* edits directories
* @param nnTopology
* @param federation
* @param conf
* @throws IOException
*/
public static void configureNameNodes(MiniDFSNNTopology nnTopology, boolean federation,
Configuration conf) throws IOException {
Preconditions.checkArgument(nnTopology.countNameNodes() > 0,
"empty NN topology: no namenodes specified!");
@ -884,22 +1004,21 @@ public class MiniDFSCluster {
// NN is started.
conf.set(FS_DEFAULT_NAME_KEY, "hdfs://127.0.0.1:" + onlyNN.getIpcPort());
}
List<String> allNsIds = Lists.newArrayList();
for (MiniDFSNNTopology.NSConf nameservice : nnTopology.getNameservices()) {
if (nameservice.getId() != null) {
allNsIds.add(nameservice.getId());
}
}
if (!allNsIds.isEmpty()) {
conf.set(DFS_NAMESERVICES, Joiner.on(",").join(allNsIds));
}
int nnCounter = 0;
for (MiniDFSNNTopology.NSConf nameservice : nnTopology.getNameservices()) {
String nsId = nameservice.getId();
String lastDefaultFileSystem = null;
Preconditions.checkArgument(
!federation || nsId != null,
"if there is more than one NS, they must have names");
@ -918,85 +1037,10 @@ public class MiniDFSCluster {
// If HA is enabled on this nameservice, enumerate all the namenodes
// in the configuration. Also need to set a shared edits dir
if (nnIds.size() > 1) {
conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, nameservice.getId()),
Joiner.on(",").join(nnIds));
if (manageNameDfsSharedDirs) {
URI sharedEditsUri = getSharedEditsDir(nnCounter, nnCounter+nnIds.size()-1);
conf.set(DFS_NAMENODE_SHARED_EDITS_DIR_KEY, sharedEditsUri.toString());
// Clean out the shared edits dir completely, including all subdirectories.
FileUtil.fullyDelete(new File(sharedEditsUri));
}
}
// Now format first NN and copy the storage directory from that node to the others.
int i = 0;
Collection<URI> prevNNDirs = null;
int nnCounterForFormat = nnCounter;
for (NNConf nn : nameservice.getNNs()) {
initNameNodeConf(conf, nsId, nn.getNnId(), manageNameDfsDirs, manageNameDfsDirs,
nnCounterForFormat);
Collection<URI> namespaceDirs = FSNamesystem.getNamespaceDirs(conf);
if (format) {
for (URI nameDirUri : namespaceDirs) {
File nameDir = new File(nameDirUri);
if (nameDir.exists() && !FileUtil.fullyDelete(nameDir)) {
throw new IOException("Could not fully delete " + nameDir);
}
}
Collection<URI> checkpointDirs = Util.stringCollectionAsURIs(conf
.getTrimmedStringCollection(DFS_NAMENODE_CHECKPOINT_DIR_KEY));
for (URI checkpointDirUri : checkpointDirs) {
File checkpointDir = new File(checkpointDirUri);
if (checkpointDir.exists() && !FileUtil.fullyDelete(checkpointDir)) {
throw new IOException("Could not fully delete " + checkpointDir);
}
}
}
boolean formatThisOne = format;
if (format && i++ > 0) {
// Don't format the second NN in an HA setup - that
// would result in it having a different clusterID,
// block pool ID, etc. Instead, copy the name dirs
// from the first one.
formatThisOne = false;
assert (null != prevNNDirs);
copyNameDirs(prevNNDirs, namespaceDirs, conf);
}
nnCounterForFormat++;
if (formatThisOne) {
// Allow overriding clusterID for specific NNs to test
// misconfiguration.
if (nn.getClusterId() == null) {
StartupOption.FORMAT.setClusterId(clusterId);
} else {
StartupOption.FORMAT.setClusterId(nn.getClusterId());
}
DFSTestUtil.formatNameNode(conf);
}
prevNNDirs = namespaceDirs;
}
// Start all Namenodes
for (NNConf nn : nameservice.getNNs()) {
initNameNodeConf(conf, nsId, nn.getNnId(), manageNameDfsDirs,
enableManagedDfsDirsRedundancy, nnCounter);
createNameNode(nnCounter, conf, numDataNodes, false, operation,
clusterId, nsId, nn.getNnId());
// Record the last namenode uri
if (nameNodes[nnCounter] != null && nameNodes[nnCounter].conf != null) {
lastDefaultFileSystem =
nameNodes[nnCounter].conf.get(FS_DEFAULT_NAME_KEY);
}
nnCounter++;
}
if (!federation && lastDefaultFileSystem != null) {
// Set the default file system to the actual bind address of NN.
conf.set(FS_DEFAULT_NAME_KEY, lastDefaultFileSystem);
conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, nameservice.getId()), Joiner
.on(",").join(nnIds));
}
}
}
public URI getSharedEditsDir(int minNN, int maxNN) throws IOException {
@ -1010,39 +1054,92 @@ public class MiniDFSCluster {
}
public NameNodeInfo[] getNameNodeInfos() {
return this.nameNodes;
return this.namenodes.values().toArray(new NameNodeInfo[0]);
}
private void initNameNodeConf(Configuration conf,
String nameserviceId, String nnId,
boolean manageNameDfsDirs, boolean enableManagedDfsDirsRedundancy,
int nnIndex) throws IOException {
/**
* @param nsIndex index of the namespace id to check
* @return all the namenodes bound to the given namespace index
*/
public NameNodeInfo[] getNameNodeInfos(int nsIndex) {
int i = 0;
for (String ns : this.namenodes.keys()) {
if (i++ == nsIndex) {
return this.namenodes.get(ns).toArray(new NameNodeInfo[0]);
}
}
return null;
}
/**
* @param nameservice id of nameservice to read
* @return all the namenodes bound to the given namespace index
*/
public NameNodeInfo[] getNameNodeInfos(String nameservice) {
for (String ns : this.namenodes.keys()) {
if (nameservice.equals(ns)) {
return this.namenodes.get(ns).toArray(new NameNodeInfo[0]);
}
}
return null;
}
private void initNameNodeConf(Configuration conf, String nameserviceId, int nsIndex, String nnId,
boolean manageNameDfsDirs, boolean enableManagedDfsDirsRedundancy, int nnIndex)
throws IOException {
if (nameserviceId != null) {
conf.set(DFS_NAMESERVICE_ID, nameserviceId);
}
if (nnId != null) {
conf.set(DFS_HA_NAMENODE_ID_KEY, nnId);
}
if (manageNameDfsDirs) {
if (enableManagedDfsDirsRedundancy) {
conf.set(DFS_NAMENODE_NAME_DIR_KEY,
fileAsURI(new File(base_dir, "name" + (2*nnIndex + 1)))+","+
fileAsURI(new File(base_dir, "name" + (2*nnIndex + 2))));
conf.set(DFS_NAMENODE_CHECKPOINT_DIR_KEY,
fileAsURI(new File(base_dir, "namesecondary" + (2*nnIndex + 1)))+","+
fileAsURI(new File(base_dir, "namesecondary" + (2*nnIndex + 2))));
File[] files = getNameNodeDirectory(nsIndex, nnIndex);
conf.set(DFS_NAMENODE_NAME_DIR_KEY, fileAsURI(files[0]) + "," + fileAsURI(files[1]));
files = getCheckpointDirectory(nsIndex, nnIndex);
conf.set(DFS_NAMENODE_CHECKPOINT_DIR_KEY, fileAsURI(files[0]) + "," + fileAsURI(files[1]));
} else {
conf.set(DFS_NAMENODE_NAME_DIR_KEY,
fileAsURI(new File(base_dir, "name" + (2*nnIndex + 1))).
toString());
conf.set(DFS_NAMENODE_CHECKPOINT_DIR_KEY,
fileAsURI(new File(base_dir, "namesecondary" + (2*nnIndex + 1))).
toString());
File[] files = getNameNodeDirectory(nsIndex, nnIndex);
conf.set(DFS_NAMENODE_NAME_DIR_KEY, fileAsURI(files[0]).toString());
files = getCheckpointDirectory(nsIndex, nnIndex);
conf.set(DFS_NAMENODE_CHECKPOINT_DIR_KEY, fileAsURI(files[0]).toString());
}
}
}
private File[] getNameNodeDirectory(int nameserviceIndex, int nnIndex) {
return getNameNodeDirectory(base_dir, nameserviceIndex, nnIndex);
}
public static File[] getNameNodeDirectory(String base_dir, int nsIndex, int nnIndex) {
return getNameNodeDirectory(new File(base_dir), nsIndex, nnIndex);
}
public static File[] getNameNodeDirectory(File base_dir, int nsIndex, int nnIndex) {
File[] files = new File[2];
files[0] = new File(base_dir, "name-" + nsIndex + "-" + (2 * nnIndex + 1));
files[1] = new File(base_dir, "name-" + nsIndex + "-" + (2 * nnIndex + 2));
return files;
}
public File[] getCheckpointDirectory(int nsIndex, int nnIndex) {
return getCheckpointDirectory(base_dir, nsIndex, nnIndex);
}
public static File[] getCheckpointDirectory(String base_dir, int nsIndex, int nnIndex) {
return getCheckpointDirectory(new File(base_dir), nsIndex, nnIndex);
}
public static File[] getCheckpointDirectory(File base_dir, int nsIndex, int nnIndex) {
File[] files = new File[2];
files[0] = new File(base_dir, "namesecondary-" + nsIndex + "-" + (2 * nnIndex + 1));
files[1] = new File(base_dir, "namesecondary-" + nsIndex + "-" + (2 * nnIndex + 2));
return files;
}
public static void copyNameDirs(Collection<URI> srcDirs, Collection<URI> dstDirs,
Configuration dstConf) throws IOException {
URI srcDir = Lists.newArrayList(srcDirs).get(0);
@ -1094,12 +1191,9 @@ public class MiniDFSCluster {
new String[] {} : new String[] {operation.getName()};
return args;
}
private void createNameNode(int nnIndex, Configuration conf,
int numDataNodes, boolean format, StartupOption operation,
String clusterId, String nameserviceId,
String nnId)
throws IOException {
private NameNodeInfo createNameNode(Configuration conf, boolean format, StartupOption operation,
String clusterId, String nameserviceId, String nnId) throws IOException {
// Format and clean out DataNode directories
if (format) {
DFSTestUtil.formatNameNode(conf);
@ -1113,7 +1207,7 @@ public class MiniDFSCluster {
String[] args = createArgs(operation);
NameNode nn = NameNode.createNameNode(args, conf);
if (operation == StartupOption.RECOVER) {
return;
return null;
}
// After the NN has started, set back the bound ports into
@ -1131,14 +1225,17 @@ public class MiniDFSCluster {
DFSUtil.setGenericConf(conf, nameserviceId, nnId,
DFS_NAMENODE_HTTP_ADDRESS_KEY);
nameNodes[nnIndex] = new NameNodeInfo(nn, nameserviceId, nnId,
NameNodeInfo info = new NameNodeInfo(nn, nameserviceId, nnId,
operation, new Configuration(conf));
namenodes.put(nameserviceId, info);
// Restore the default fs name
if (originalDefaultFs == null) {
conf.set(FS_DEFAULT_NAME_KEY, "");
} else {
conf.set(FS_DEFAULT_NAME_KEY, originalDefaultFs);
}
return info;
}
/**
@ -1154,7 +1251,7 @@ public class MiniDFSCluster {
*/
public URI getURI(int nnIndex) {
String hostPort =
nameNodes[nnIndex].nameNode.getNameNodeAddressHostPortString();
getNN(nnIndex).nameNode.getNameNodeAddressHostPortString();
URI uri = null;
try {
uri = new URI("hdfs://" + hostPort);
@ -1172,9 +1269,21 @@ public class MiniDFSCluster {
* @return Configuration of for the given namenode
*/
public Configuration getConfiguration(int nnIndex) {
return nameNodes[nnIndex].conf;
return getNN(nnIndex).conf;
}
private NameNodeInfo getNN(int nnIndex) {
int count = 0;
for (NameNodeInfo nn : namenodes.values()) {
if (count == nnIndex) {
return nn;
}
count++;
}
return null;
}
/**
* wait for the given namenode to get out of safemode.
*/
@ -1593,7 +1702,7 @@ public class MiniDFSCluster {
* @throws Exception
*/
public void finalizeCluster(int nnIndex, Configuration conf) throws Exception {
finalizeNamenode(nameNodes[nnIndex].nameNode, nameNodes[nnIndex].conf);
finalizeNamenode(getNN(nnIndex).nameNode, getNN(nnIndex).conf);
}
/**
@ -1604,7 +1713,7 @@ public class MiniDFSCluster {
* @throws IllegalStateException if the Namenode is not running.
*/
public void finalizeCluster(Configuration conf) throws Exception {
for (NameNodeInfo nnInfo : nameNodes) {
for (NameNodeInfo nnInfo : namenodes.values()) {
if (nnInfo == null) {
throw new IllegalStateException("Attempting to finalize "
+ "Namenode but it is not running");
@ -1612,9 +1721,9 @@ public class MiniDFSCluster {
finalizeNamenode(nnInfo.nameNode, nnInfo.conf);
}
}
public int getNumNameNodes() {
return nameNodes.length;
return namenodes.size();
}
/**
@ -1644,7 +1753,7 @@ public class MiniDFSCluster {
* Gets the NameNode for the index. May be null.
*/
public NameNode getNameNode(int nnIndex) {
return nameNodes[nnIndex].nameNode;
return getNN(nnIndex).nameNode;
}
/**
@ -1653,11 +1762,11 @@ public class MiniDFSCluster {
*/
public FSNamesystem getNamesystem() {
checkSingleNameNode();
return NameNodeAdapter.getNamesystem(nameNodes[0].nameNode);
return NameNodeAdapter.getNamesystem(getNN(0).nameNode);
}
public FSNamesystem getNamesystem(int nnIndex) {
return NameNodeAdapter.getNamesystem(nameNodes[nnIndex].nameNode);
return NameNodeAdapter.getNamesystem(getNN(nnIndex).nameNode);
}
/**
@ -1697,14 +1806,14 @@ public class MiniDFSCluster {
* caller supplied port is not necessarily the actual port used.
*/
public int getNameNodePort(int nnIndex) {
return nameNodes[nnIndex].nameNode.getNameNodeAddress().getPort();
return getNN(nnIndex).nameNode.getNameNodeAddress().getPort();
}
/**
* @return the service rpc port used by the NameNode at the given index.
*/
public int getNameNodeServicePort(int nnIndex) {
return nameNodes[nnIndex].nameNode.getServiceRpcAddress().getPort();
return getNN(nnIndex).nameNode.getServiceRpcAddress().getPort();
}
/**
@ -1745,7 +1854,7 @@ public class MiniDFSCluster {
fileSystems.clear();
}
shutdownDataNodes();
for (NameNodeInfo nnInfo : nameNodes) {
for (NameNodeInfo nnInfo : namenodes.values()) {
if (nnInfo == null) continue;
NameNode nameNode = nnInfo.nameNode;
if (nameNode != null) {
@ -1781,7 +1890,7 @@ public class MiniDFSCluster {
* Shutdown all the namenodes.
*/
public synchronized void shutdownNameNodes() {
for (int i = 0; i < nameNodes.length; i++) {
for (int i = 0; i < namenodes.size(); i++) {
shutdownNameNode(i);
}
}
@ -1790,13 +1899,15 @@ public class MiniDFSCluster {
* Shutdown the namenode at a given index.
*/
public synchronized void shutdownNameNode(int nnIndex) {
NameNode nn = nameNodes[nnIndex].nameNode;
NameNodeInfo info = getNN(nnIndex);
NameNode nn = info.nameNode;
if (nn != null) {
LOG.info("Shutting down the namenode");
nn.stop();
nn.join();
Configuration conf = nameNodes[nnIndex].conf;
nameNodes[nnIndex] = new NameNodeInfo(null, null, null, null, conf);
info.nnId = null;
info.nameNode = null;
info.nameserviceId = null;
}
}
@ -1804,7 +1915,7 @@ public class MiniDFSCluster {
* Restart all namenodes.
*/
public synchronized void restartNameNodes() throws IOException {
for (int i = 0; i < nameNodes.length; i++) {
for (int i = 0; i < namenodes.size(); i++) {
restartNameNode(i, false);
}
waitActive();
@ -1840,19 +1951,19 @@ public class MiniDFSCluster {
*/
public synchronized void restartNameNode(int nnIndex, boolean waitActive,
String... args) throws IOException {
String nameserviceId = nameNodes[nnIndex].nameserviceId;
String nnId = nameNodes[nnIndex].nnId;
StartupOption startOpt = nameNodes[nnIndex].startOpt;
Configuration conf = nameNodes[nnIndex].conf;
NameNodeInfo info = getNN(nnIndex);
StartupOption startOpt = info.startOpt;
shutdownNameNode(nnIndex);
if (args.length != 0) {
startOpt = null;
} else {
args = createArgs(startOpt);
}
NameNode nn = NameNode.createNameNode(args, conf);
nameNodes[nnIndex] = new NameNodeInfo(nn, nameserviceId, nnId, startOpt,
conf);
NameNode nn = NameNode.createNameNode(args, info.conf);
info.nameNode = nn;
info.setStartOpt(startOpt);
if (waitActive) {
waitClusterUp();
LOG.info("Restarted the namenode");
@ -2124,7 +2235,7 @@ public class MiniDFSCluster {
* or if waiting for safe mode is disabled.
*/
public boolean isNameNodeUp(int nnIndex) {
NameNode nameNode = nameNodes[nnIndex].nameNode;
NameNode nameNode = getNN(nnIndex).nameNode;
if (nameNode == null) {
return false;
}
@ -2142,7 +2253,7 @@ public class MiniDFSCluster {
* Returns true if all the NameNodes are running and is out of Safe Mode.
*/
public boolean isClusterUp() {
for (int index = 0; index < nameNodes.length; index++) {
for (int index = 0; index < namenodes.size(); index++) {
if (!isNameNodeUp(index)) {
return false;
}
@ -2172,15 +2283,13 @@ public class MiniDFSCluster {
checkSingleNameNode();
return getFileSystem(0);
}
/**
* Get a client handle to the DFS cluster for the namenode at given index.
*/
public DistributedFileSystem getFileSystem(int nnIndex) throws IOException {
DistributedFileSystem dfs = (DistributedFileSystem) FileSystem.get(
getURI(nnIndex), nameNodes[nnIndex].conf);
fileSystems.add(dfs);
return dfs;
return (DistributedFileSystem) addFileSystem(FileSystem.get(getURI(nnIndex),
getNN(nnIndex).conf));
}
/**
@ -2188,17 +2297,20 @@ public class MiniDFSCluster {
* This simulating different threads working on different FileSystem instances.
*/
public FileSystem getNewFileSystemInstance(int nnIndex) throws IOException {
FileSystem dfs = FileSystem.newInstance(getURI(nnIndex), nameNodes[nnIndex].conf);
fileSystems.add(dfs);
return dfs;
return addFileSystem(FileSystem.newInstance(getURI(nnIndex), getNN(nnIndex).conf));
}
private <T extends FileSystem> T addFileSystem(T fs) {
fileSystems.add(fs);
return fs;
}
/**
* @return a http URL
*/
public String getHttpUri(int nnIndex) {
return "http://"
+ nameNodes[nnIndex].conf
+ getNN(nnIndex).conf
.get(DFS_NAMENODE_HTTP_ADDRESS_KEY);
}
@ -2206,14 +2318,14 @@ public class MiniDFSCluster {
* Get the directories where the namenode stores its image.
*/
public Collection<URI> getNameDirs(int nnIndex) {
return FSNamesystem.getNamespaceDirs(nameNodes[nnIndex].conf);
return FSNamesystem.getNamespaceDirs(getNN(nnIndex).conf);
}
/**
* Get the directories where the namenode stores its edits.
*/
public Collection<URI> getNameEditsDirs(int nnIndex) throws IOException {
return FSNamesystem.getNamespaceEditsDirs(nameNodes[nnIndex].conf);
return FSNamesystem.getNamespaceEditsDirs(getNN(nnIndex).conf);
}
public void transitionToActive(int nnIndex) throws IOException,
@ -2254,11 +2366,12 @@ public class MiniDFSCluster {
/** Wait until the given namenode gets registration from all the datanodes */
public void waitActive(int nnIndex) throws IOException {
if (nameNodes.length == 0 || nameNodes[nnIndex] == null
|| nameNodes[nnIndex].nameNode == null) {
if (namenodes.size() == 0 || getNN(nnIndex) == null || getNN(nnIndex).nameNode == null) {
return;
}
InetSocketAddress addr = nameNodes[nnIndex].nameNode.getServiceRpcAddress();
NameNodeInfo info = getNN(nnIndex);
InetSocketAddress addr = info.nameNode.getServiceRpcAddress();
assert addr.getPort() != 0;
DFSClient client = new DFSClient(addr, conf);
@ -2278,7 +2391,7 @@ public class MiniDFSCluster {
* Wait until the cluster is active and running.
*/
public void waitActive() throws IOException {
for (int index = 0; index < nameNodes.length; index++) {
for (int index = 0; index < namenodes.size(); index++) {
int failedCount = 0;
while (true) {
try {
@ -2298,7 +2411,14 @@ public class MiniDFSCluster {
}
LOG.info("Cluster is active");
}
public void printNNs() {
for (int i = 0; i < namenodes.size(); i++) {
LOG.info("Have namenode " + i + ", info:" + getNN(i));
LOG.info(" has namenode: " + getNN(i).nameNode);
}
}
private synchronized boolean shouldWait(DatanodeInfo[] dnInfo,
InetSocketAddress addr) {
// If a datanode failed to start, then do not wait
@ -2696,7 +2816,7 @@ public class MiniDFSCluster {
* namenode
*/
private void checkSingleNameNode() {
if (nameNodes.length != 1) {
if (namenodes.size() != 1) {
throw new IllegalArgumentException("Namenode index is needed");
}
}
@ -2712,13 +2832,9 @@ public class MiniDFSCluster {
if(!federation)
throw new IOException("cannot add namenode to non-federated cluster");
int nnIndex = nameNodes.length;
int numNameNodes = nameNodes.length + 1;
NameNodeInfo[] newlist = new NameNodeInfo[numNameNodes];
System.arraycopy(nameNodes, 0, newlist, 0, nameNodes.length);
nameNodes = newlist;
String nameserviceId = NAMESERVICE_ID_PREFIX + (nnIndex + 1);
int nameServiceIndex = namenodes.keys().size();
String nameserviceId = NAMESERVICE_ID_PREFIX + (namenodes.keys().size() + 1);
String nameserviceIds = conf.get(DFS_NAMESERVICES);
nameserviceIds += "," + nameserviceId;
conf.set(DFS_NAMESERVICES, nameserviceIds);
@ -2726,9 +2842,11 @@ public class MiniDFSCluster {
String nnId = null;
initNameNodeAddress(conf, nameserviceId,
new NNConf(nnId).setIpcPort(namenodePort));
initNameNodeConf(conf, nameserviceId, nnId, true, true, nnIndex);
createNameNode(nnIndex, conf, numDataNodes, true, null, null,
nameserviceId, nnId);
// figure out the current number of NNs
NameNodeInfo[] infos = this.getNameNodeInfos(nameserviceId);
int nnIndex = infos == null ? 0 : infos.length;
initNameNodeConf(conf, nameserviceId, nameServiceIndex, nnId, true, true, nnIndex);
NameNodeInfo info = createNameNode(conf, true, null, null, nameserviceId, nnId);
// Refresh datanodes with the newly started namenode
for (DataNodeProperties dn : dataNodes) {
@ -2738,7 +2856,7 @@ public class MiniDFSCluster {
// Wait for new namenode to get registrations from all the datanodes
waitActive(nnIndex);
return nameNodes[nnIndex].nameNode;
return info.nameNode;
}
protected void setupDatanodeAddress(Configuration conf, boolean setupHostsFile,

18
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java
View File

@ -56,10 +56,20 @@ public class MiniDFSNNTopology {
* Set up an HA topology with a single HA nameservice.
*/
public static MiniDFSNNTopology simpleHATopology() {
return new MiniDFSNNTopology()
.addNameservice(new MiniDFSNNTopology.NSConf("minidfs-ns")
.addNN(new MiniDFSNNTopology.NNConf("nn1"))
.addNN(new MiniDFSNNTopology.NNConf("nn2")));
return simpleHATopology(2);
}
/**
* Set up an HA topology with a single HA nameservice.
* @param nnCount of namenodes to use with the nameservice
*/
public static MiniDFSNNTopology simpleHATopology(int nnCount) {
MiniDFSNNTopology.NSConf nameservice = new MiniDFSNNTopology.NSConf("minidfs-ns");
for (int i = 1; i <= nnCount; i++) {
nameservice.addNN(new MiniDFSNNTopology.NNConf("nn" + i));
}
MiniDFSNNTopology topology = new MiniDFSNNTopology().addNameservice(nameservice);
return topology;
}
/**

6
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
View File

@ -303,12 +303,12 @@ public class TestDFSUpgradeFromImage {
unpackStorage(HADOOP22_IMAGE, HADOOP_DFS_DIR_TXT);
// Overwrite the md5 stored in the VERSION files
File baseDir = new File(MiniDFSCluster.getBaseDirectory());
File[] nnDirs = MiniDFSCluster.getNameNodeDirectory(MiniDFSCluster.getBaseDirectory(), 0, 0);
FSImageTestUtil.corruptVersionFile(
new File(baseDir, "name1/current/VERSION"),
new File(nnDirs[0], "current/VERSION"),
"imageMD5Digest", "22222222222222222222222222222222");
FSImageTestUtil.corruptVersionFile(
new File(baseDir, "name2/current/VERSION"),
new File(nnDirs[1], "current/VERSION"),
"imageMD5Digest", "22222222222222222222222222222222");
// Attach our own log appender so we can verify output

170
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java
View File

@ -19,8 +19,18 @@ package org.apache.hadoop.hdfs;
import java.io.File;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.util.concurrent.ThreadLocalRandom;
import javax.management.AttributeNotFoundException;
import javax.management.InstanceNotFoundException;
import javax.management.MBeanException;
import javax.management.MBeanServer;
import javax.management.MalformedObjectNameException;
import javax.management.ObjectName;
import javax.management.ReflectionException;
import javax.management.openmbean.CompositeDataSupport;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@ -45,6 +55,9 @@ import org.apache.hadoop.io.IOUtils;
import org.junit.Assert;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNull;
/**
* This class tests rolling upgrade.
@ -55,7 +68,7 @@ public class TestRollingUpgrade {
public static void runCmd(DFSAdmin dfsadmin, boolean success,
String... args) throws Exception {
if (success) {
Assert.assertEquals(0, dfsadmin.run(args));
assertEquals(0, dfsadmin.run(args));
} else {
Assert.assertTrue(dfsadmin.run(args) != 0);
}
@ -66,7 +79,7 @@ public class TestRollingUpgrade {
*/
@Test
public void testDFSAdminRollingUpgradeCommands() throws Exception {
// start a cluster
// start a cluster
final Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = null;
try {
@ -85,6 +98,7 @@ public class TestRollingUpgrade {
//illegal argument "abc" to rollingUpgrade option
runCmd(dfsadmin, false, "-rollingUpgrade", "abc");
checkMxBeanIsNull();
//query rolling upgrade
runCmd(dfsadmin, true, "-rollingUpgrade");
@ -95,11 +109,16 @@ public class TestRollingUpgrade {
//query rolling upgrade
runCmd(dfsadmin, true, "-rollingUpgrade", "query");
checkMxBean();
dfs.mkdirs(bar);
//finalize rolling upgrade
runCmd(dfsadmin, true, "-rollingUpgrade", "finalize");
// RollingUpgradeInfo should be null after finalization, both via
// Java API and in JMX
assertNull(dfs.rollingUpgrade(RollingUpgradeAction.QUERY));
checkMxBeanIsNull();
dfs.mkdirs(baz);
@ -143,7 +162,7 @@ public class TestRollingUpgrade {
String nnDirPrefix = MiniDFSCluster.getBaseDirectory() + "/nn/";
final File nn1Dir = new File(nnDirPrefix + "image1");
final File nn2Dir = new File(nnDirPrefix + "image2");
LOG.info("nn1Dir=" + nn1Dir);
LOG.info("nn2Dir=" + nn2Dir);
@ -186,9 +205,9 @@ public class TestRollingUpgrade {
final RollingUpgradeInfo info1;
{
final DistributedFileSystem dfs = cluster.getFileSystem();
final DistributedFileSystem dfs = cluster.getFileSystem();
dfs.mkdirs(foo);
//start rolling upgrade
dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
info1 = dfs.rollingUpgrade(RollingUpgradeAction.PREPARE);
@ -196,8 +215,8 @@ public class TestRollingUpgrade {
LOG.info("START\n" + info1);
//query rolling upgrade
Assert.assertEquals(info1, dfs.rollingUpgrade(RollingUpgradeAction.QUERY));
assertEquals(info1, dfs.rollingUpgrade(RollingUpgradeAction.QUERY));
dfs.mkdirs(bar);
cluster.shutdown();
}
@ -209,21 +228,21 @@ public class TestRollingUpgrade {
.format(false)
.manageNameDfsDirs(false)
.build();
final DistributedFileSystem dfs2 = cluster2.getFileSystem();
final DistributedFileSystem dfs2 = cluster2.getFileSystem();
// Check that cluster2 sees the edits made on cluster1
Assert.assertTrue(dfs2.exists(foo));
Assert.assertTrue(dfs2.exists(bar));
Assert.assertFalse(dfs2.exists(baz));
//query rolling upgrade in cluster2
Assert.assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY));
assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY));
dfs2.mkdirs(baz);
LOG.info("RESTART cluster 2");
cluster2.restartNameNode();
Assert.assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY));
assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY));
Assert.assertTrue(dfs2.exists(foo));
Assert.assertTrue(dfs2.exists(bar));
Assert.assertTrue(dfs2.exists(baz));
@ -237,7 +256,7 @@ public class TestRollingUpgrade {
LOG.info("RESTART cluster 2 again");
cluster2.restartNameNode();
Assert.assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY));
assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY));
Assert.assertTrue(dfs2.exists(foo));
Assert.assertTrue(dfs2.exists(bar));
Assert.assertTrue(dfs2.exists(baz));
@ -258,9 +277,31 @@ public class TestRollingUpgrade {
}
}
private static CompositeDataSupport getBean()
throws MalformedObjectNameException, MBeanException,
AttributeNotFoundException, InstanceNotFoundException,
ReflectionException {
MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
ObjectName mxbeanName =
new ObjectName("Hadoop:service=NameNode,name=NameNodeInfo");
return (CompositeDataSupport)mbs.getAttribute(mxbeanName,
"RollingUpgradeStatus");
}
private static void checkMxBeanIsNull() throws Exception {
CompositeDataSupport ruBean = getBean();
assertNull(ruBean);
}
private static void checkMxBean() throws Exception {
CompositeDataSupport ruBean = getBean();
assertNotEquals(0l, ruBean.get("startTime"));
assertEquals(0l, ruBean.get("finalizeTime"));
}
@Test
public void testRollback() throws IOException {
// start a cluster
public void testRollback() throws Exception {
// start a cluster
final Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = null;
try {
@ -278,10 +319,13 @@ public class TestRollingUpgrade {
out.write(data, 0, data.length);
out.close();
checkMxBeanIsNull();
startRollingUpgrade(foo, bar, file, data, cluster);
checkMxBean();
cluster.getFileSystem().rollEdits();
cluster.getFileSystem().rollEdits();
rollbackRollingUpgrade(foo, bar, file, data, cluster);
checkMxBeanIsNull();
startRollingUpgrade(foo, bar, file, data, cluster);
cluster.getFileSystem().rollEdits();
@ -305,7 +349,7 @@ public class TestRollingUpgrade {
if(cluster != null) cluster.shutdown();
}
}
private static void startRollingUpgrade(Path foo, Path bar,
Path file, byte[] data,
MiniDFSCluster cluster) throws IOException {
@ -327,7 +371,7 @@ public class TestRollingUpgrade {
TestFileTruncate.checkBlockRecovery(file, dfs);
AppendTestUtil.checkFullFile(dfs, file, newLength, data);
}
private static void rollbackRollingUpgrade(Path foo, Path bar,
Path file, byte[] data,
MiniDFSCluster cluster) throws IOException {
@ -355,39 +399,50 @@ public class TestRollingUpgrade {
// check the datanode
final String dnAddr = dn.getDatanodeId().getIpcAddr(false);
final String[] args1 = {"-getDatanodeInfo", dnAddr};
Assert.assertEquals(0, dfsadmin.run(args1));
runCmd(dfsadmin, true, args1);
// issue shutdown to the datanode.
final String[] args2 = {"-shutdownDatanode", dnAddr, "upgrade" };
Assert.assertEquals(0, dfsadmin.run(args2));
runCmd(dfsadmin, true, args2);
// the datanode should be down.
Thread.sleep(2000);
Assert.assertFalse("DataNode should exit", dn.isDatanodeUp());
// ping should fail.
Assert.assertEquals(-1, dfsadmin.run(args1));
assertEquals(-1, dfsadmin.run(args1));
} finally {
if (cluster != null) cluster.shutdown();
}
}
@Test (timeout = 300000)
@Test(timeout = 300000)
public void testFinalize() throws Exception {
testFinalize(2);
}
@Test(timeout = 300000)
public void testFinalizeWithMultipleNN() throws Exception {
testFinalize(3);
}
private void testFinalize(int nnCount) throws Exception {
final Configuration conf = new HdfsConfiguration();
MiniQJMHACluster cluster = null;
final Path foo = new Path("/foo");
final Path bar = new Path("/bar");
try {
cluster = new MiniQJMHACluster.Builder(conf).build();
cluster = new MiniQJMHACluster.Builder(conf).setNumNameNodes(nnCount).build();
MiniDFSCluster dfsCluster = cluster.getDfsCluster();
dfsCluster.waitActive();
// let NN1 tail editlog every 1s
dfsCluster.getConfiguration(1).setInt(
DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
dfsCluster.restartNameNode(1);
// let other NN tail editlog every 1s
for(int i=1; i < nnCount; i++) {
dfsCluster.getConfiguration(i).setInt(
DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
}
dfsCluster.restartNameNodes();
dfsCluster.transitionToActive(0);
DistributedFileSystem dfs = dfsCluster.getFileSystem(0);
@ -425,17 +480,29 @@ public class TestRollingUpgrade {
@Test (timeout = 300000)
public void testQuery() throws Exception {
testQuery(2);
}
@Test (timeout = 300000)
public void testQueryWithMultipleNN() throws Exception {
testQuery(3);
}
private void testQuery(int nnCount) throws Exception{
final Configuration conf = new Configuration();
MiniQJMHACluster cluster = null;
try {
cluster = new MiniQJMHACluster.Builder(conf).build();
cluster = new MiniQJMHACluster.Builder(conf).setNumNameNodes(nnCount).build();
MiniDFSCluster dfsCluster = cluster.getDfsCluster();
dfsCluster.waitActive();
dfsCluster.transitionToActive(0);
DistributedFileSystem dfs = dfsCluster.getFileSystem(0);
dfsCluster.shutdownNameNode(1);
// shutdown other NNs
for (int i = 1; i < nnCount; i++) {
dfsCluster.shutdownNameNode(i);
}
// start rolling upgrade
RollingUpgradeInfo info = dfs
@ -445,13 +512,16 @@ public class TestRollingUpgrade {
info = dfs.rollingUpgrade(RollingUpgradeAction.QUERY);
Assert.assertFalse(info.createdRollbackImages());
dfsCluster.restartNameNode(1);
// restart other NNs
for (int i = 1; i < nnCount; i++) {
dfsCluster.restartNameNode(i);
}
// check that one of the other NNs has created the rollback image and uploaded it
queryForPreparation(dfs);
// The NN should have a copy of the fsimage in case of rollbacks.
Assert.assertTrue(dfsCluster.getNamesystem(0).getFSImage()
.hasRollbackFSImage());
.hasRollbackFSImage());
} finally {
if (cluster != null) {
cluster.shutdown();
@ -487,6 +557,15 @@ public class TestRollingUpgrade {
@Test(timeout = 300000)
public void testCheckpoint() throws IOException, InterruptedException {
testCheckpoint(2);
}
@Test(timeout = 300000)
public void testCheckpointWithMultipleNN() throws IOException, InterruptedException {
testCheckpoint(3);
}
public void testCheckpoint(int nnCount) throws IOException, InterruptedException {
final Configuration conf = new Configuration();
conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY, 1);
@ -495,7 +574,7 @@ public class TestRollingUpgrade {
final Path foo = new Path("/foo");
try {
cluster = new MiniQJMHACluster.Builder(conf).build();
cluster = new MiniQJMHACluster.Builder(conf).setNumNameNodes(nnCount).build();
MiniDFSCluster dfsCluster = cluster.getDfsCluster();
dfsCluster.waitActive();
@ -513,16 +592,9 @@ public class TestRollingUpgrade {
long txid = dfs.rollEdits();
Assert.assertTrue(txid > 0);
int retries = 0;
while (++retries < 5) {
NNStorage storage = dfsCluster.getNamesystem(1).getFSImage()
.getStorage();
if (storage.getFsImageName(txid - 1) != null) {
return;
}
Thread.sleep(1000);
for(int i=1; i< nnCount; i++) {
verifyNNCheckpoint(dfsCluster, txid, i);
}
Assert.fail("new checkpoint does not exist");
} finally {
if (cluster != null) {
@ -531,6 +603,22 @@ public class TestRollingUpgrade {
}
}
/**
* Verify that the namenode at the given index has an FSImage with a TxId up to txid-1
*/
private void verifyNNCheckpoint(MiniDFSCluster dfsCluster, long txid, int nnIndex) throws InterruptedException {
int retries = 0;
while (++retries < 5) {
NNStorage storage = dfsCluster.getNamesystem(nnIndex).getFSImage()
.getStorage();
if (storage.getFsImageName(txid - 1) != null) {
return;
}
Thread.sleep(1000);
}
Assert.fail("new checkpoint does not exist");
}
static void queryForPreparation(DistributedFileSystem dfs) throws IOException,
InterruptedException {
RollingUpgradeInfo info;

94
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java
View File

@ -17,43 +17,39 @@
*/
package org.apache.hadoop.hdfs.qjournal;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY;
import java.io.IOException;
import java.net.BindException;
import java.net.URI;
import java.util.Random;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import java.io.IOException;
import java.net.BindException;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
public class MiniQJMHACluster {
private MiniDFSCluster cluster;
private MiniJournalCluster journalCluster;
private final Configuration conf;
private static final Log LOG = LogFactory.getLog(MiniQJMHACluster.class);
public static final String NAMESERVICE = "ns1";
private static final String NN1 = "nn1";
private static final String NN2 = "nn2";
private static final Random RANDOM = new Random();
private int basePort = 10000;
public static class Builder {
private final Configuration conf;
private StartupOption startOpt = null;
private int numNNs = 2;
private final MiniDFSCluster.Builder dfsBuilder;
public Builder(Configuration conf) {
this.conf = conf;
// most QJMHACluster tests don't need DataNodes, so we'll make
@ -64,7 +60,7 @@ public class MiniQJMHACluster {
public MiniDFSCluster.Builder getDfsBuilder() {
return dfsBuilder;
}
public MiniQJMHACluster build() throws IOException {
return new MiniQJMHACluster(this);
}
@ -72,15 +68,25 @@ public class MiniQJMHACluster {
public void startupOption(StartupOption startOpt) {
this.startOpt = startOpt;
}
public Builder setNumNameNodes(int nns) {
this.numNNs = nns;
return this;
}
}
public static MiniDFSNNTopology createDefaultTopology(int nns, int startingPort) {
MiniDFSNNTopology.NSConf nameservice = new MiniDFSNNTopology.NSConf(NAMESERVICE);
for (int i = 0; i < nns; i++) {
nameservice.addNN(new MiniDFSNNTopology.NNConf("nn" + i).setIpcPort(startingPort++)
.setHttpPort(startingPort++));
}
return new MiniDFSNNTopology().addNameservice(nameservice);
}
public static MiniDFSNNTopology createDefaultTopology(int basePort) {
return new MiniDFSNNTopology()
.addNameservice(new MiniDFSNNTopology.NSConf(NAMESERVICE).addNN(
new MiniDFSNNTopology.NNConf("nn1").setIpcPort(basePort)
.setHttpPort(basePort + 1)).addNN(
new MiniDFSNNTopology.NNConf("nn2").setIpcPort(basePort + 2)
.setHttpPort(basePort + 3)));
return createDefaultTopology(2, basePort);
}
private MiniQJMHACluster(Builder builder) throws IOException {
@ -94,10 +100,10 @@ public class MiniQJMHACluster {
.build();
URI journalURI = journalCluster.getQuorumJournalURI(NAMESERVICE);
// start cluster with 2 NameNodes
MiniDFSNNTopology topology = createDefaultTopology(basePort);
// start cluster with specified NameNodes
MiniDFSNNTopology topology = createDefaultTopology(builder.numNNs, basePort);
initHAConf(journalURI, builder.conf);
initHAConf(journalURI, builder.conf, builder.numNNs);
// First start up the NNs just to format the namespace. The MinIDFSCluster
// has no way to just format the NameNodes without also starting them.
@ -110,8 +116,9 @@ public class MiniQJMHACluster {
Configuration confNN0 = cluster.getConfiguration(0);
NameNode.initializeSharedEdits(confNN0, true);
cluster.getNameNodeInfos()[0].setStartOpt(builder.startOpt);
cluster.getNameNodeInfos()[1].setStartOpt(builder.startOpt);
for (MiniDFSCluster.NameNodeInfo nn : cluster.getNameNodeInfos()) {
nn.setStartOpt(builder.startOpt);
}
// restart the cluster
cluster.restartNameNodes();
@ -123,31 +130,28 @@ public class MiniQJMHACluster {
}
}
}
private Configuration initHAConf(URI journalURI, Configuration conf) {
private Configuration initHAConf(URI journalURI, Configuration conf, int numNNs) {
conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY,
journalURI.toString());
String address1 = "127.0.0.1:" + basePort;
String address2 = "127.0.0.1:" + (basePort + 2);
conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY,
NAMESERVICE, NN1), address1);
conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY,
NAMESERVICE, NN2), address2);
conf.set(DFSConfigKeys.DFS_NAMESERVICES, NAMESERVICE);
conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, NAMESERVICE),
NN1 + "," + NN2);
conf.set(HdfsClientConfigKeys.Failover.PROXY_PROVIDER_KEY_PREFIX + "." + NAMESERVICE,
ConfiguredFailoverProxyProvider.class.getName());
conf.set("fs.defaultFS", "hdfs://" + NAMESERVICE);
List<String> nns = new ArrayList<String>(numNNs);
int port = basePort;
for (int i = 0; i < numNNs; i++) {
nns.add("127.0.0.1:" + port);
// increment by 2 each time to account for the http port in the config setting
port += 2;
}
// use standard failover configurations
HATestUtil.setFailoverConfigurations(conf, NAMESERVICE, nns);
return conf;
}
public MiniDFSCluster getDfsCluster() {
return cluster;
}
public MiniJournalCluster getJournalCluster() {
return journalCluster;
}

10
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
View File

@ -162,7 +162,7 @@ public class TestBlockToken {
public void testWritable() throws Exception {
TestWritable.testWritable(new BlockTokenIdentifier());
BlockTokenSecretManager sm = new BlockTokenSecretManager(
blockKeyUpdateInterval, blockTokenLifetime, 0, "fake-pool", null);
blockKeyUpdateInterval, blockTokenLifetime, 0, 1, "fake-pool", null);
TestWritable.testWritable(generateTokenId(sm, block1,
EnumSet.allOf(BlockTokenIdentifier.AccessMode.class)));
TestWritable.testWritable(generateTokenId(sm, block2,
@ -201,7 +201,7 @@ public class TestBlockToken {
@Test
public void testBlockTokenSecretManager() throws Exception {
BlockTokenSecretManager masterHandler = new BlockTokenSecretManager(
blockKeyUpdateInterval, blockTokenLifetime, 0, "fake-pool", null);
blockKeyUpdateInterval, blockTokenLifetime, 0, 1, "fake-pool", null);
BlockTokenSecretManager slaveHandler = new BlockTokenSecretManager(
blockKeyUpdateInterval, blockTokenLifetime, "fake-pool", null);
ExportedBlockKeys keys = masterHandler.exportKeys();
@ -244,7 +244,7 @@ public class TestBlockToken {
UserGroupInformation.setConfiguration(conf);
BlockTokenSecretManager sm = new BlockTokenSecretManager(
blockKeyUpdateInterval, blockTokenLifetime, 0, "fake-pool", null);
blockKeyUpdateInterval, blockTokenLifetime, 0, 1, "fake-pool", null);
Token<BlockTokenIdentifier> token = sm.generateToken(block3,
EnumSet.allOf(BlockTokenIdentifier.AccessMode.class));
@ -283,7 +283,7 @@ public class TestBlockToken {
Assume.assumeTrue(FD_DIR.exists());
BlockTokenSecretManager sm = new BlockTokenSecretManager(
blockKeyUpdateInterval, blockTokenLifetime, 0, "fake-pool", null);
blockKeyUpdateInterval, blockTokenLifetime, 0, 1, "fake-pool", null);
Token<BlockTokenIdentifier> token = sm.generateToken(block3,
EnumSet.allOf(BlockTokenIdentifier.AccessMode.class));
@ -352,7 +352,7 @@ public class TestBlockToken {
for (int i = 0; i < 10; i++) {
String bpid = Integer.toString(i);
BlockTokenSecretManager masterHandler = new BlockTokenSecretManager(
blockKeyUpdateInterval, blockTokenLifetime, 0, "fake-pool", null);
blockKeyUpdateInterval, blockTokenLifetime, 0, 1, "fake-pool", null);
BlockTokenSecretManager slaveHandler = new BlockTokenSecretManager(
blockKeyUpdateInterval, blockTokenLifetime, "fake-pool", null);
bpMgr.addBlockPool(bpid, slaveHandler);

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
View File

@ -63,7 +63,7 @@ public class TestBlockInfo {
final DatanodeStorageInfo storage = DFSTestUtil.createDatanodeStorageInfo("storageID", "127.0.0.1");
boolean added = blockInfo.addStorage(storage);
boolean added = blockInfo.addStorage(storage, blockInfo);
Assert.assertTrue(added);
Assert.assertEquals(storage, blockInfo.getStorageInfo(0));

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
View File

@ -383,7 +383,7 @@ public class TestBlockManager {
for (int i = 1; i < pipeline.length; i++) {
DatanodeStorageInfo storage = pipeline[i];
bm.addBlock(storage, blockInfo, null);
blockInfo.addStorage(storage);
blockInfo.addStorage(storage, blockInfo);
}
}
@ -393,7 +393,7 @@ public class TestBlockManager {
for (DatanodeDescriptor dn : nodes) {
for (DatanodeStorageInfo storage : dn.getStorageInfos()) {
blockInfo.addStorage(storage);
blockInfo.addStorage(storage, blockInfo);
}
}
return blockInfo;

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java
View File

@ -100,7 +100,7 @@ public class TestNodeCount {
DatanodeDescriptor nonExcessDN = null;
for(DatanodeStorageInfo storage : bm.blocksMap.getStorages(block.getLocalBlock())) {
final DatanodeDescriptor dn = storage.getDatanodeDescriptor();
Collection<Block> blocks = bm.excessReplicateMap.get(dn.getDatanodeUuid());
Collection<BlockInfo> blocks = bm.excessReplicateMap.get(dn.getDatanodeUuid());
if (blocks == null || !blocks.contains(block.getLocalBlock()) ) {
nonExcessDN = dn;
break;

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java
View File

@ -34,7 +34,6 @@ import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
@ -42,7 +41,6 @@ import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.util.Time;
import org.junit.Test;
public class TestOverReplicatedBlocks {
@ -185,7 +183,7 @@ public class TestOverReplicatedBlocks {
// All replicas for deletion should be scheduled on lastDN.
// And should not actually be deleted, because lastDN does not heartbeat.
namesystem.readLock();
Collection<Block> dnBlocks =
Collection<BlockInfo> dnBlocks =
namesystem.getBlockManager().excessReplicateMap.get(lastDNid);
assertEquals("Replicas on node " + lastDNid + " should have been deleted",
SMALL_FILE_LENGTH / SMALL_BLOCK_SIZE, dnBlocks.size());

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
View File

@ -1250,7 +1250,7 @@ public class TestReplicationPolicy {
when(storage.removeBlock(any(BlockInfo.class))).thenReturn(true);
when(storage.addBlock(any(BlockInfo.class))).thenReturn
(DatanodeStorageInfo.AddBlockResult.ADDED);
ucBlock.addStorage(storage);
ucBlock.addStorage(storage, ucBlock);
when(mbc.setLastBlock((BlockInfo) any(), (DatanodeStorageInfo[]) any()))
.thenReturn(ucBlock);

11
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java
View File

@ -23,6 +23,7 @@ import java.io.File;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@ -53,6 +54,16 @@ public class DataNodeTestUtils {
dn.setHeartbeatsDisabledForTests(heartbeatsDisabledForTests);
}
/**
* Set if cache reports are disabled for all DNs in a mini cluster.
*/
public static void setCacheReportsDisabledForTests(MiniDFSCluster cluster,
boolean disabled) {
for (DataNode dn : cluster.getDataNodes()) {
dn.setCacheReportsDisabledForTest(disabled);
}
}
public static void triggerDeletionReport(DataNode dn) throws IOException {
for (BPOfferService bpos : dn.getAllBpOs()) {
bpos.triggerDeletionReportForTests();

28
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestRbwSpaceReservation.java
View File

@ -49,11 +49,15 @@ import org.mockito.Mockito;
import java.io.IOException;
import java.io.OutputStream;
import java.lang.management.ManagementFactory;
import java.lang.reflect.Field;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.TimeoutException;
import javax.management.MBeanServer;
import javax.management.ObjectName;
/**
* Ensure that the DN reserves disk space equivalent to a full block for
* replica being written (RBW).
@ -324,6 +328,30 @@ public class TestRbwSpaceReservation {
fsVolumeImpl.getReservedForRbw() == 0);
}
@Test(timeout = 30000)
public void testRBWInJMXBean() throws Exception {
final short replication = 1;
startCluster(BLOCK_SIZE, replication, -1);
final String methodName = GenericTestUtils.getMethodName();
final Path file = new Path("/" + methodName + ".01.dat");
try (FSDataOutputStream os = fs.create(file, replication)) {
// Write 1 byte to the file
os.write(new byte[1]);
os.hsync();
final MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
final ObjectName mxbeanName = new ObjectName(
"Hadoop:service=DataNode,name=DataNodeInfo");
final String volumeInfo = (String) mbs.getAttribute(mxbeanName,
"VolumeInfo");
assertTrue(volumeInfo.contains("reservedSpaceForRBW"));
}
}
/**
* Stress test to ensure we are not leaking reserved space.
* @throws IOException

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
View File

@ -319,7 +319,7 @@ public class TestBackupNode {
if(fileSys != null) fileSys.close();
if(cluster != null) cluster.shutdown();
}
File nnCurDir = new File(BASE_DIR, "name1/current/");
File nnCurDir = new File(MiniDFSCluster.getNameNodeDirectory(BASE_DIR, 0, 0)[0], "current/");
File bnCurDir = new File(getBackupNodeDir(op, 1), "/current/");
FSImageTestUtil.assertParallelFilesAreIdentical(

25
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
View File

@ -76,6 +76,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.CachedBlocksList.Type;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.io.nativeio.NativeIO.POSIX.CacheManipulator;
@ -1510,4 +1511,28 @@ public class TestCacheDirectives {
Thread.sleep(1000);
checkPendingCachedEmpty(cluster);
}
@Test(timeout=60000)
public void testNoBackingReplica() throws Exception {
// Cache all three replicas for a file.
final Path filename = new Path("/noback");
final short replication = (short) 3;
DFSTestUtil.createFile(dfs, filename, 1, replication, 0x0BAC);
dfs.addCachePool(new CachePoolInfo("pool"));
dfs.addCacheDirective(
new CacheDirectiveInfo.Builder().setPool("pool").setPath(filename)
.setReplication(replication).build());
waitForCachedBlocks(namenode, 1, replication, "testNoBackingReplica:1");
// Pause cache reports while we change the replication factor.
// This will orphan some cached replicas.
DataNodeTestUtils.setCacheReportsDisabledForTests(cluster, true);
try {
dfs.setReplication(filename, (short) 1);
DFSTestUtil.waitForReplication(dfs, filename, (short) 1, 30000);
// The cache locations should drop down to 1 even without cache reports.
waitForCachedBlocks(namenode, 1, (short) 1, "testNoBackingReplica:2");
} finally {
DataNodeTestUtils.setCacheReportsDisabledForTests(cluster, false);
}
}
}

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
View File

@ -1428,7 +1428,8 @@ public class TestCheckpoint {
//
secondary = startSecondaryNameNode(conf);
File secondaryDir = new File(MiniDFSCluster.getBaseDirectory(), "namesecondary1");
File secondaryDir = MiniDFSCluster.getCheckpointDirectory(MiniDFSCluster.getBaseDirectory(),
0, 0)[0];
File secondaryCurrent = new File(secondaryDir, "current");
long expectedTxIdToDownload = cluster.getNameNode().getFSImage()

5
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
View File

@ -28,7 +28,6 @@ import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.namenode.top.TopConf;
import org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.io.nativeio.NativeIO.POSIX.NoMlockCacheManipulator;
import org.apache.hadoop.util.VersionInfo;
@ -46,8 +45,6 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager.Op;
import static org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager.TopWindow;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
@ -197,6 +194,8 @@ public class TestNameNodeMXBean {
assertEquals(NativeIO.POSIX.getCacheManipulator().getMemlockLimit() *
cluster.getDataNodes().size(),
mbs.getAttribute(mxbeanName, "CacheCapacity"));
assertNull("RollingUpgradeInfo should be null when there is no rolling"
+ " upgrade", mbs.getAttribute(mxbeanName, "RollingUpgradeStatus"));
} finally {
if (cluster != null) {
for (URI dir : cluster.getNameDirs(0)) {

46
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HAStressTestHarness.java
View File

@ -42,7 +42,8 @@ public class HAStressTestHarness {
private MiniDFSCluster cluster;
static final int BLOCK_SIZE = 1024;
final TestContext testCtx = new TestContext();
private int nns = 2;
public HAStressTestHarness() {
conf = new Configuration();
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
@ -54,12 +55,20 @@ public class HAStressTestHarness {
DFSConfigKeys.DFS_NAMENODE_REPLICATION_STREAMS_HARD_LIMIT_KEY, 16);
}
/**
* Set the number of namenodes that should be run. This must be set before calling
* {@link #startCluster()}
*/
public void setNumberOfNameNodes(int nns) {
this.nns = nns;
}
/**
* Start and return the MiniDFSCluster.
*/
public MiniDFSCluster startCluster() throws IOException {
cluster = new MiniDFSCluster.Builder(conf)
.nnTopology(MiniDFSNNTopology.simpleHATopology())
.nnTopology(MiniDFSNNTopology.simpleHATopology(nns))
.numDataNodes(3)
.build();
return cluster;
@ -99,28 +108,27 @@ public class HAStressTestHarness {
}
/**
* Add a thread which periodically triggers failover back and forth between
* the two namenodes.
* Add a thread which periodically triggers failover back and forth between the namenodes.
*/
public void addFailoverThread(final int msBetweenFailovers) {
testCtx.addThread(new RepeatingTestThread(testCtx) {
@Override
public void doAnAction() throws Exception {
System.err.println("==============================\n" +
"Failing over from 0->1\n" +
"==================================");
cluster.transitionToStandby(0);
cluster.transitionToActive(1);
Thread.sleep(msBetweenFailovers);
System.err.println("==============================\n" +
"Failing over from 1->0\n" +
"==================================");
cluster.transitionToStandby(1);
cluster.transitionToActive(0);
Thread.sleep(msBetweenFailovers);
// fail over from one namenode to the next, all the way back to the original NN
for (int i = 0; i < nns; i++) {
// next node, mod nns so we wrap to the 0th NN on the last iteration
int next = (i + 1) % nns;
System.err.println("==============================\n"
+ "[Starting] Failing over from " + i + "->" + next + "\n"
+ "==============================");
cluster.transitionToStandby(i);
cluster.transitionToActive(next);
System.err.println("==============================\n"
+ "[Completed] Failing over from " + i + "->" + next + ". Sleeping for "+
(msBetweenFailovers/1000) +"sec \n"
+ "==============================");
Thread.sleep(msBetweenFailovers);
}
}
});
}

60
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java
View File

@ -24,9 +24,14 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.TimeoutException;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.collect.Iterables;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@ -67,12 +72,11 @@ public abstract class HATestUtil {
*/
public static void waitForStandbyToCatchUp(NameNode active,
NameNode standby) throws InterruptedException, IOException, CouldNotCatchUpException {
long activeTxId = active.getNamesystem().getFSImage().getEditLog()
.getLastWrittenTxId();
active.getRpcServer().rollEditLog();
long start = Time.now();
while (Time.now() - start < TestEditLogTailer.NN_LAG_TIMEOUT) {
long nn2HighestTxId = standby.getNamesystem().getFSImage()
@ -166,34 +170,52 @@ public abstract class HATestUtil {
/** Sets the required configurations for performing failover. */
public static void setFailoverConfigurations(MiniDFSCluster cluster,
Configuration conf, String logicalName, int nsIndex) {
InetSocketAddress nnAddr1 = cluster.getNameNode(2 * nsIndex).getNameNodeAddress();
InetSocketAddress nnAddr2 = cluster.getNameNode(2 * nsIndex + 1).getNameNodeAddress();
setFailoverConfigurations(conf, logicalName, nnAddr1, nnAddr2);
MiniDFSCluster.NameNodeInfo[] nns = cluster.getNameNodeInfos(nsIndex);
List<InetSocketAddress> nnAddresses = new ArrayList<InetSocketAddress>(3);
for (MiniDFSCluster.NameNodeInfo nn : nns) {
nnAddresses.add(nn.nameNode.getNameNodeAddress());
}
setFailoverConfigurations(conf, logicalName, nnAddresses);
}
public static void setFailoverConfigurations(Configuration conf, String logicalName,
InetSocketAddress ... nnAddresses){
setFailoverConfigurations(conf, logicalName, Arrays.asList(nnAddresses));
}
/**
* Sets the required configurations for performing failover
*/
public static void setFailoverConfigurations(Configuration conf,
String logicalName, InetSocketAddress nnAddr1,
InetSocketAddress nnAddr2) {
String nameNodeId1 = "nn1";
String nameNodeId2 = "nn2";
String address1 = "hdfs://" + nnAddr1.getHostName() + ":" + nnAddr1.getPort();
String address2 = "hdfs://" + nnAddr2.getHostName() + ":" + nnAddr2.getPort();
conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY,
logicalName, nameNodeId1), address1);
conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY,
logicalName, nameNodeId2), address2);
String logicalName, List<InetSocketAddress> nnAddresses) {
setFailoverConfigurations(conf, logicalName,
Iterables.transform(nnAddresses, new Function<InetSocketAddress, String>() {
// transform the inet address to a simple string
@Override
public String apply(InetSocketAddress addr) {
return "hdfs://" + addr.getHostName() + ":" + addr.getPort();
}
}));
}
public static void setFailoverConfigurations(Configuration conf, String logicalName,
Iterable<String> nnAddresses) {
List<String> nnids = new ArrayList<String>();
int i = 0;
for (String address : nnAddresses) {
String nnId = "nn" + (i + 1);
nnids.add(nnId);
conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, logicalName, nnId), address);
i++;
}
conf.set(DFSConfigKeys.DFS_NAMESERVICES, logicalName);
conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, logicalName),
nameNodeId1 + "," + nameNodeId2);
Joiner.on(',').join(nnids));
conf.set(HdfsClientConfigKeys.Failover.PROXY_PROVIDER_KEY_PREFIX + "." + logicalName,
ConfiguredFailoverProxyProvider.class.getName());
conf.set("fs.defaultFS", "hdfs://" + logicalName);
}
public static String getLogicalHostname(MiniDFSCluster cluster) {
return String.format(LOGICAL_HOSTNAME, cluster.getInstanceId());

178
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java
View File

@ -46,37 +46,47 @@ import com.google.common.collect.ImmutableList;
public class TestBootstrapStandby {
private static final Log LOG = LogFactory.getLog(TestBootstrapStandby.class);
private static final int maxNNCount = 3;
private static final int STARTING_PORT = 20000;
private MiniDFSCluster cluster;
private NameNode nn0;
@Before
public void setupCluster() throws IOException {
Configuration conf = new Configuration();
MiniDFSNNTopology topology = new MiniDFSNNTopology()
.addNameservice(new MiniDFSNNTopology.NSConf("ns1")
.addNN(new MiniDFSNNTopology.NNConf("nn1").setHttpPort(20001))
.addNN(new MiniDFSNNTopology.NNConf("nn2").setHttpPort(20002)));
// duplicate code with MiniQJMHACluster#createDefaultTopology, but don't want to cross
// dependencies or munge too much code to support it all correctly
MiniDFSNNTopology.NSConf nameservice = new MiniDFSNNTopology.NSConf("ns1");
for (int i = 0; i < maxNNCount; i++) {
nameservice.addNN(new MiniDFSNNTopology.NNConf("nn" + i).setHttpPort(STARTING_PORT + i + 1));
}
MiniDFSNNTopology topology = new MiniDFSNNTopology().addNameservice(nameservice);
cluster = new MiniDFSCluster.Builder(conf)
.nnTopology(topology)
.numDataNodes(0)
.build();
.nnTopology(topology)
.numDataNodes(0)
.build();
cluster.waitActive();
nn0 = cluster.getNameNode(0);
cluster.transitionToActive(0);
cluster.shutdownNameNode(1);
// shutdown the other NNs
for (int i = 1; i < maxNNCount; i++) {
cluster.shutdownNameNode(i);
}
}
@After
public void shutdownCluster() {
if (cluster != null) {
cluster.shutdown();
}
}
/**
* Test for the base success case. The primary NN
* hasn't made any checkpoints, and we copy the fsimage_0
@ -85,30 +95,29 @@ public class TestBootstrapStandby {
@Test
public void testSuccessfulBaseCase() throws Exception {
removeStandbyNameDirs();
try {
cluster.restartNameNode(1);
fail("Did not throw");
} catch (IOException ioe) {
GenericTestUtils.assertExceptionContains(
"storage directory does not exist or is not accessible",
ioe);
}
int rc = BootstrapStandby.run(
new String[]{"-nonInteractive"},
cluster.getConfiguration(1));
assertEquals(0, rc);
// Should have copied over the namespace from the active
FSImageTestUtil.assertNNHasCheckpoints(cluster, 1,
ImmutableList.of(0));
FSImageTestUtil.assertNNFilesMatch(cluster);
// We should now be able to start the standby successfully.
cluster.restartNameNode(1);
// skip the first NN, its up
for (int index = 1; index < maxNNCount; index++) {
try {
cluster.restartNameNode(index);
fail("Did not throw");
} catch (IOException ioe) {
GenericTestUtils.assertExceptionContains(
"storage directory does not exist or is not accessible", ioe);
}
int rc = BootstrapStandby.run(new String[] { "-nonInteractive" },
cluster.getConfiguration(index));
assertEquals(0, rc);
// Should have copied over the namespace from the active
FSImageTestUtil.assertNNHasCheckpoints(cluster, index, ImmutableList.of(0));
}
// We should now be able to start the standbys successfully.
restartNameNodesFromIndex(1);
}
/**
* Test for downloading a checkpoint made at a later checkpoint
* from the active.
@ -123,21 +132,21 @@ public class TestBootstrapStandby {
NameNodeAdapter.saveNamespace(nn0);
NameNodeAdapter.leaveSafeMode(nn0);
long expectedCheckpointTxId = NameNodeAdapter.getNamesystem(nn0)
.getFSImage().getMostRecentCheckpointTxId();
.getFSImage().getMostRecentCheckpointTxId();
assertEquals(6, expectedCheckpointTxId);
int rc = BootstrapStandby.run(
new String[]{"-force"},
cluster.getConfiguration(1));
assertEquals(0, rc);
// Should have copied over the namespace from the active
FSImageTestUtil.assertNNHasCheckpoints(cluster, 1,
ImmutableList.of((int)expectedCheckpointTxId));
for (int i = 1; i < maxNNCount; i++) {
assertEquals(0, forceBootstrap(i));
// Should have copied over the namespace from the active
LOG.info("Checking namenode: " + i);
FSImageTestUtil.assertNNHasCheckpoints(cluster, i,
ImmutableList.of((int) expectedCheckpointTxId));
}
FSImageTestUtil.assertNNFilesMatch(cluster);
// We should now be able to start the standby successfully.
cluster.restartNameNode(1);
restartNameNodesFromIndex(1);
}
/**
@ -147,36 +156,40 @@ public class TestBootstrapStandby {
@Test
public void testSharedEditsMissingLogs() throws Exception {
removeStandbyNameDirs();
CheckpointSignature sig = nn0.getRpcServer().rollEditLog();
assertEquals(3, sig.getCurSegmentTxId());
// Should have created edits_1-2 in shared edits dir
URI editsUri = cluster.getSharedEditsDir(0, 1);
URI editsUri = cluster.getSharedEditsDir(0, maxNNCount - 1);
File editsDir = new File(editsUri);
File editsSegment = new File(new File(editsDir, "current"),
File currentDir = new File(editsDir, "current");
File editsSegment = new File(currentDir,
NNStorage.getFinalizedEditsFileName(1, 2));
GenericTestUtils.assertExists(editsSegment);
GenericTestUtils.assertExists(currentDir);
// Delete the segment.
assertTrue(editsSegment.delete());
// Trying to bootstrap standby should now fail since the edit
// logs aren't available in the shared dir.
LogCapturer logs = GenericTestUtils.LogCapturer.captureLogs(
LogFactory.getLog(BootstrapStandby.class));
try {
int rc = BootstrapStandby.run(
new String[]{"-force"},
cluster.getConfiguration(1));
assertEquals(BootstrapStandby.ERR_CODE_LOGS_UNAVAILABLE, rc);
assertEquals(BootstrapStandby.ERR_CODE_LOGS_UNAVAILABLE, forceBootstrap(1));
} finally {
logs.stopCapturing();
}
GenericTestUtils.assertMatches(logs.getOutput(),
"FATAL.*Unable to read transaction ids 1-3 from the configured shared");
}
/**
* Show that bootstrapping will fail on a given NameNode if its directories already exist. Its not
* run across all the NN because its testing the state local on each node.
* @throws Exception on unexpected failure
*/
@Test
public void testStandbyDirsAlreadyExist() throws Exception {
// Should not pass since standby dirs exist, force not given
@ -186,12 +199,9 @@ public class TestBootstrapStandby {
assertEquals(BootstrapStandby.ERR_CODE_ALREADY_FORMATTED, rc);
// Should pass with -force
rc = BootstrapStandby.run(
new String[]{"-force"},
cluster.getConfiguration(1));
assertEquals(0, rc);
assertEquals(0, forceBootstrap(1));
}
/**
* Test that, even if the other node is not active, we are able
* to bootstrap standby from it.
@ -199,18 +209,44 @@ public class TestBootstrapStandby {
@Test(timeout=30000)
public void testOtherNodeNotActive() throws Exception {
cluster.transitionToStandby(0);
int rc = BootstrapStandby.run(
new String[]{"-force"},
cluster.getConfiguration(1));
assertEquals(0, rc);
assertSuccessfulBootstrapFromIndex(1);
}
private void removeStandbyNameDirs() {
for (URI u : cluster.getNameDirs(1)) {
assertTrue(u.getScheme().equals("file"));
File dir = new File(u.getPath());
LOG.info("Removing standby dir " + dir);
assertTrue(FileUtil.fullyDelete(dir));
for (int i = 1; i < maxNNCount; i++) {
for (URI u : cluster.getNameDirs(i)) {
assertTrue(u.getScheme().equals("file"));
File dir = new File(u.getPath());
LOG.info("Removing standby dir " + dir);
assertTrue(FileUtil.fullyDelete(dir));
}
}
}
}
private void restartNameNodesFromIndex(int start) throws IOException {
for (int i = start; i < maxNNCount; i++) {
// We should now be able to start the standby successfully.
cluster.restartNameNode(i, false);
}
cluster.waitClusterUp();
cluster.waitActive();
}
/**
* Force boot strapping on a namenode
* @param i index of the namenode to attempt
* @return exit code
* @throws Exception on unexpected failure
*/
private int forceBootstrap(int i) throws Exception {
return BootstrapStandby.run(new String[] { "-force" },
cluster.getConfiguration(i));
}
private void assertSuccessfulBootstrapFromIndex(int start) throws Exception {
for (int i = start; i < maxNNCount; i++) {
assertEquals(0, forceBootstrap(i));
}
}
}

47
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandbyWithQJM.java
View File

@ -52,7 +52,8 @@ public class TestBootstrapStandbyWithQJM {
private MiniDFSCluster cluster;
private MiniJournalCluster jCluster;
private int nnCount = 3;
@Before
public void setup() throws Exception {
Configuration conf = new Configuration();
@ -62,7 +63,8 @@ public class TestBootstrapStandbyWithQJM {
CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY,
0);
MiniQJMHACluster miniQjmHaCluster = new MiniQJMHACluster.Builder(conf).build();
MiniQJMHACluster miniQjmHaCluster =
new MiniQJMHACluster.Builder(conf).setNumNameNodes(nnCount).build();
cluster = miniQjmHaCluster.getDfsCluster();
jCluster = miniQjmHaCluster.getJournalCluster();
@ -90,18 +92,7 @@ public class TestBootstrapStandbyWithQJM {
public void testBootstrapStandbyWithStandbyNN() throws Exception {
// make the first NN in standby state
cluster.transitionToStandby(0);
Configuration confNN1 = cluster.getConfiguration(1);
// shut down nn1
cluster.shutdownNameNode(1);
int rc = BootstrapStandby.run(new String[] { "-force" }, confNN1);
assertEquals(0, rc);
// Should have copied over the namespace from the standby
FSImageTestUtil.assertNNHasCheckpoints(cluster, 1,
ImmutableList.of(0));
FSImageTestUtil.assertNNFilesMatch(cluster);
bootstrapStandbys();
}
/** BootstrapStandby when the existing NN is active */
@ -109,17 +100,23 @@ public class TestBootstrapStandbyWithQJM {
public void testBootstrapStandbyWithActiveNN() throws Exception {
// make the first NN in active state
cluster.transitionToActive(0);
Configuration confNN1 = cluster.getConfiguration(1);
// shut down nn1
cluster.shutdownNameNode(1);
int rc = BootstrapStandby.run(new String[] { "-force" }, confNN1);
assertEquals(0, rc);
// Should have copied over the namespace from the standby
FSImageTestUtil.assertNNHasCheckpoints(cluster, 1,
ImmutableList.of(0));
bootstrapStandbys();
}
private void bootstrapStandbys() throws Exception {
// shutdown and bootstrap all the other nns, except the first (start 1, not 0)
for (int i = 1; i < nnCount; i++) {
Configuration otherNNConf = cluster.getConfiguration(i);
// shut down other nn
cluster.shutdownNameNode(i);
int rc = BootstrapStandby.run(new String[] { "-force" }, otherNNConf);
assertEquals(0, rc);
// Should have copied over the namespace from the standby
FSImageTestUtil.assertNNHasCheckpoints(cluster, i, ImmutableList.of(0));
}
FSImageTestUtil.assertNNFilesMatch(cluster);
}

1
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencingWithReplication.java
View File

@ -107,6 +107,7 @@ public class TestDNFencingWithReplication {
@Test
public void testFencingStress() throws Exception {
HAStressTestHarness harness = new HAStressTestHarness();
harness.setNumberOfNameNodes(3);
harness.conf.setInt(
DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000);
harness.conf.setInt(

12
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java
View File

@ -113,7 +113,12 @@ public class TestEditLogTailer {
public void testNN1TriggersLogRolls() throws Exception {
testStandbyTriggersLogRolls(1);
}
@Test
public void testNN2TriggersLogRolls() throws Exception {
testStandbyTriggersLogRolls(2);
}
private static void testStandbyTriggersLogRolls(int activeIndex)
throws Exception {
Configuration conf = new Configuration();
@ -125,7 +130,8 @@ public class TestEditLogTailer {
MiniDFSNNTopology topology = new MiniDFSNNTopology()
.addNameservice(new MiniDFSNNTopology.NSConf("ns1")
.addNN(new MiniDFSNNTopology.NNConf("nn1").setIpcPort(10031))
.addNN(new MiniDFSNNTopology.NNConf("nn2").setIpcPort(10032)));
.addNN(new MiniDFSNNTopology.NNConf("nn2").setIpcPort(10032))
.addNN(new MiniDFSNNTopology.NNConf("nn3").setIpcPort(10033)));
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.nnTopology(topology)
@ -145,7 +151,7 @@ public class TestEditLogTailer {
private static void waitForLogRollInSharedDir(MiniDFSCluster cluster,
long startTxId) throws Exception {
URI sharedUri = cluster.getSharedEditsDir(0, 1);
URI sharedUri = cluster.getSharedEditsDir(0, 2);
File sharedDir = new File(sharedUri.getPath(), "current");
final File expectedLog = new File(sharedDir,
NNStorage.getInProgressEditsFileName(startTxId));

55
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailoverWithBlockTokensEnabled.java
View File

@ -56,10 +56,11 @@ public class TestFailoverWithBlockTokensEnabled {
private static final Path TEST_PATH = new Path("/test-path");
private static final String TEST_DATA = "very important text";
private static final int numNNs = 3;
private Configuration conf;
private MiniDFSCluster cluster;
@Before
public void startCluster() throws IOException {
conf = new Configuration();
@ -67,7 +68,7 @@ public class TestFailoverWithBlockTokensEnabled {
// Set short retry timeouts so this test runs faster
conf.setInt(HdfsClientConfigKeys.Retry.WINDOW_BASE_KEY, 10);
cluster = new MiniDFSCluster.Builder(conf)
.nnTopology(MiniDFSNNTopology.simpleHATopology())
.nnTopology(MiniDFSNNTopology.simpleHATopology(numNNs))
.numDataNodes(1)
.build();
}
@ -78,33 +79,41 @@ public class TestFailoverWithBlockTokensEnabled {
cluster.shutdown();
}
}
@Test
public void ensureSerialNumbersNeverOverlap() {
BlockTokenSecretManager btsm1 = cluster.getNamesystem(0).getBlockManager()
.getBlockTokenSecretManager();
BlockTokenSecretManager btsm2 = cluster.getNamesystem(1).getBlockManager()
.getBlockTokenSecretManager();
btsm1.setSerialNo(0);
btsm2.setSerialNo(0);
assertFalse(btsm1.getSerialNoForTesting() == btsm2.getSerialNoForTesting());
btsm1.setSerialNo(Integer.MAX_VALUE);
btsm2.setSerialNo(Integer.MAX_VALUE);
assertFalse(btsm1.getSerialNoForTesting() == btsm2.getSerialNoForTesting());
btsm1.setSerialNo(Integer.MIN_VALUE);
btsm2.setSerialNo(Integer.MIN_VALUE);
assertFalse(btsm1.getSerialNoForTesting() == btsm2.getSerialNoForTesting());
btsm1.setSerialNo(Integer.MAX_VALUE / 2);
btsm2.setSerialNo(Integer.MAX_VALUE / 2);
assertFalse(btsm1.getSerialNoForTesting() == btsm2.getSerialNoForTesting());
BlockTokenSecretManager btsm3 = cluster.getNamesystem(2).getBlockManager()
.getBlockTokenSecretManager();
btsm1.setSerialNo(Integer.MIN_VALUE / 2);
btsm2.setSerialNo(Integer.MIN_VALUE / 2);
assertFalse(btsm1.getSerialNoForTesting() == btsm2.getSerialNoForTesting());
setAndCheckSerialNumber(0, btsm1, btsm2, btsm3);
setAndCheckSerialNumber(Integer.MAX_VALUE, btsm1, btsm2, btsm3);
setAndCheckSerialNumber(Integer.MIN_VALUE, btsm1, btsm2, btsm3);
setAndCheckSerialNumber(Integer.MAX_VALUE / 2, btsm1, btsm2, btsm3);
setAndCheckSerialNumber(Integer.MIN_VALUE / 2, btsm1, btsm2, btsm3);
setAndCheckSerialNumber(Integer.MAX_VALUE / 3, btsm1, btsm2, btsm3);
setAndCheckSerialNumber(Integer.MIN_VALUE / 3, btsm1, btsm2, btsm3);
}
private void setAndCheckSerialNumber(int serialNumber, BlockTokenSecretManager... btsms) {
for (BlockTokenSecretManager btsm : btsms) {
btsm.setSerialNo(serialNumber);
}
for (int i = 0; i < btsms.length; i++) {
for (int j = 0; j < btsms.length; j++) {
if (j == i) {
continue;
}
int first = btsms[i].getSerialNoForTesting();
int second = btsms[j].getSerialNoForTesting();
assertFalse("Overlap found for set serial number (" + serialNumber + ") is " + i + ": "
+ first + " == " + j + ": " + second, first == second);
}
}
}
@Test

49
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAConfiguration.java
View File

@ -23,10 +23,12 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.util.Collection;
import com.google.common.base.Joiner;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
@ -58,19 +60,23 @@ public class TestHAConfiguration {
}
}
private Configuration getHAConf(String nsId, String host1, String host2) {
private Configuration getHAConf(String nsId, String ... hosts) {
Configuration conf = new Configuration();
conf.set(DFSConfigKeys.DFS_NAMESERVICES, nsId);
conf.set(DFSUtil.addKeySuffixes(
DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX, nsId),
"nn1,nn2");
conf.set(DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY, "nn1");
String[] nnids = new String[hosts.length];
for (int i = 0; i < hosts.length; i++) {
String nnid = "nn" + (i + 1);
nnids[i] = nnid;
conf.set(DFSUtil.addKeySuffixes(
DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY, nsId, nnid),
hosts[i] + ":12345");
}
conf.set(DFSUtil.addKeySuffixes(
DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY, nsId, "nn1"),
host1 + ":12345");
conf.set(DFSUtil.addKeySuffixes(
DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY, nsId, "nn2"),
host2 + ":12345");
DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX, nsId),
Joiner.on(',').join(nnids));
return conf;
}
@ -87,11 +93,28 @@ public class TestHAConfiguration {
// 0.0.0.0, it should substitute the address from the RPC configuration
// above.
StandbyCheckpointer checkpointer = new StandbyCheckpointer(conf, fsn);
assertEquals(new URL("http", "1.2.3.2",
DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT, ""),
checkpointer.getActiveNNAddress());
assertAddressMatches("1.2.3.2", checkpointer.getActiveNNAddresses().get(0));
//test when there are three NNs
// Use non-local addresses to avoid host address matching
conf = getHAConf("ns1", "1.2.3.1", "1.2.3.2", "1.2.3.3");
// This is done by the NN before the StandbyCheckpointer is created
NameNode.initializeGenericKeys(conf, "ns1", "nn1");
checkpointer = new StandbyCheckpointer(conf, fsn);
assertEquals("Got an unexpected number of possible active NNs", 2, checkpointer
.getActiveNNAddresses().size());
assertEquals(new URL("http", "1.2.3.2", DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT, ""),
checkpointer.getActiveNNAddresses().get(0));
assertAddressMatches("1.2.3.2", checkpointer.getActiveNNAddresses().get(0));
assertAddressMatches("1.2.3.3", checkpointer.getActiveNNAddresses().get(1));
}
private void assertAddressMatches(String address, URL url) throws MalformedURLException {
assertEquals(new URL("http", address, DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT, ""), url);
}
/**
* Tests that the namenode edits dirs and shared edits dirs are gotten with
* duplicates removed

110
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java
View File

@ -24,6 +24,7 @@ import static org.junit.Assert.fail;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.util.Random;
import java.util.concurrent.TimeoutException;
import org.apache.commons.logging.Log;
@ -81,24 +82,33 @@ public class TestPipelinesFailover {
private static final int STRESS_NUM_THREADS = 25;
private static final int STRESS_RUNTIME = 40000;
private static final int NN_COUNT = 3;
private static final long FAILOVER_SEED = System.currentTimeMillis();
private static final Random failoverRandom = new Random(FAILOVER_SEED);
static{
// log the failover seed so we can reproduce the test exactly
LOG.info("Using random seed: " + FAILOVER_SEED
+ " for selecting active target NN during failover");
}
enum TestScenario {
GRACEFUL_FAILOVER {
@Override
void run(MiniDFSCluster cluster) throws IOException {
cluster.transitionToStandby(0);
cluster.transitionToActive(1);
void run(MiniDFSCluster cluster, int previousActive, int activeIndex) throws IOException {
cluster.transitionToStandby(previousActive);
cluster.transitionToActive(activeIndex);
}
},
ORIGINAL_ACTIVE_CRASHED {
@Override
void run(MiniDFSCluster cluster) throws IOException {
cluster.restartNameNode(0);
cluster.transitionToActive(1);
void run(MiniDFSCluster cluster, int previousActive, int activeIndex) throws IOException {
cluster.restartNameNode(previousActive);
cluster.transitionToActive(activeIndex);
}
};
abstract void run(MiniDFSCluster cluster) throws IOException;
abstract void run(MiniDFSCluster cluster, int previousActive, int activeIndex) throws IOException;
}
enum MethodToTestIdempotence {
@ -135,10 +145,7 @@ public class TestPipelinesFailover {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1000);
FSDataOutputStream stm = null;
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.nnTopology(MiniDFSNNTopology.simpleHATopology())
.numDataNodes(3)
.build();
MiniDFSCluster cluster = newMiniCluster(conf, 3);
try {
int sizeWritten = 0;
@ -157,15 +164,15 @@ public class TestPipelinesFailover {
// Make sure all of the blocks are written out before failover.
stm.hflush();
LOG.info("Failing over to NN 1");
scenario.run(cluster);
LOG.info("Failing over to another NN");
int activeIndex = failover(cluster, scenario);
// NOTE: explicitly do *not* make any further metadata calls
// to the NN here. The next IPC call should be to allocate the next
// block. Any other call would notice the failover and not test
// idempotence of the operation (HDFS-3031)
FSNamesystem ns1 = cluster.getNameNode(1).getNamesystem();
FSNamesystem ns1 = cluster.getNameNode(activeIndex).getNamesystem();
BlockManagerTestUtil.updateState(ns1.getBlockManager());
assertEquals(0, ns1.getPendingReplicationBlocks());
assertEquals(0, ns1.getCorruptReplicaBlocks());
@ -213,10 +220,7 @@ public class TestPipelinesFailover {
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
FSDataOutputStream stm = null;
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.nnTopology(MiniDFSNNTopology.simpleHATopology())
.numDataNodes(5)
.build();
MiniDFSCluster cluster = newMiniCluster(conf, 5);
try {
cluster.waitActive();
cluster.transitionToActive(0);
@ -232,8 +236,7 @@ public class TestPipelinesFailover {
// Make sure all the blocks are written before failover
stm.hflush();
LOG.info("Failing over to NN 1");
scenario.run(cluster);
int nextActive = failover(cluster, scenario);
assertTrue(fs.exists(TEST_PATH));
@ -242,9 +245,9 @@ public class TestPipelinesFailover {
// write another block and a half
AppendTestUtil.write(stm, BLOCK_AND_A_HALF, BLOCK_AND_A_HALF);
stm.hflush();
LOG.info("Failing back to NN 0");
cluster.transitionToStandby(1);
LOG.info("Failing back from NN " + nextActive + " to NN 0");
cluster.transitionToStandby(nextActive);
cluster.transitionToActive(0);
cluster.stopDataNode(1);
@ -262,7 +265,7 @@ public class TestPipelinesFailover {
cluster.shutdown();
}
}
/**
* Tests lease recovery if a client crashes. This approximates the
* use case of HBase WALs being recovered after a NN failover.
@ -275,10 +278,7 @@ public class TestPipelinesFailover {
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
FSDataOutputStream stm = null;
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.nnTopology(MiniDFSNNTopology.simpleHATopology())
.numDataNodes(3)
.build();
final MiniDFSCluster cluster = newMiniCluster(conf, 3);
try {
cluster.waitActive();
cluster.transitionToActive(0);
@ -329,10 +329,7 @@ public class TestPipelinesFailover {
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
FSDataOutputStream stm = null;
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.nnTopology(MiniDFSNNTopology.simpleHATopology())
.numDataNodes(3)
.build();
final MiniDFSCluster cluster = newMiniCluster(conf, 3);
try {
cluster.waitActive();
cluster.transitionToActive(0);
@ -406,7 +403,20 @@ public class TestPipelinesFailover {
cluster.shutdown();
}
}
/**
* Create a MiniCluster with the specified base configuration and the specified number of
* DataNodes. Helper method to ensure that the we use the same number of NNs across all the tests.
* @return mini cluster ready to use
* @throws IOException cluster cannot be started
*/
private MiniDFSCluster newMiniCluster(Configuration conf, int dnCount) throws IOException {
return new MiniDFSCluster.Builder(conf)
.nnTopology(MiniDFSNNTopology.simpleHATopology(NN_COUNT))
.numDataNodes(dnCount)
.build();
}
/**
* Stress test for pipeline/lease recovery. Starts a number of
* threads, each of which creates a file and has another client
@ -484,6 +494,38 @@ public class TestPipelinesFailover {
}
}
/**
* Fail-over using the given scenario, assuming NN0 is currently active
* @param cluster cluster on which to run the scenario
* @param scenario failure scenario to run
* @return the index of the new active NN
* @throws IOException
*/
private int failover(MiniDFSCluster cluster, TestScenario scenario) throws IOException {
return failover(cluster, scenario, 0);
}
/**
* Do a fail-over with the given scenario.
* @param cluster cluster on which to run the scenario
* @param scenario failure scenario to run
* @param activeIndex index of the currently active node
* @throws IOException on failure
* @return the index of the new active NN
*/
private int failover(MiniDFSCluster cluster, TestScenario scenario, int activeIndex)
throws IOException {
// get index of the next node that should be active, ensuring its not the same as the currently
// active node
int nextActive = failoverRandom.nextInt(NN_COUNT);
if (nextActive == activeIndex) {
nextActive = (nextActive + 1) % NN_COUNT;
}
LOG.info("Failing over to a standby NN:" + nextActive + " from NN " + activeIndex);
scenario.run(cluster, activeIndex, nextActive);
return nextActive;
}
/**
* Test thread which creates a file, has another fake user recover
* the lease on the file, and then ensures that the file's contents

61
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRemoteNameNodeInfo.java Normal file
View File

@ -0,0 +1,61 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode.ha;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.junit.Test;
import java.util.List;
import static org.junit.Assert.assertEquals;
/**
* Test that we correctly obtain remote namenode information
*/
public class TestRemoteNameNodeInfo {
@Test
public void testParseMultipleNameNodes() throws Exception {
// start with an empty configuration
Configuration conf = new Configuration(false);
// add in keys for each of the NNs
String nameservice = "ns1";
MiniDFSNNTopology topology = new MiniDFSNNTopology()
.addNameservice(new MiniDFSNNTopology.NSConf(nameservice)
.addNN(new MiniDFSNNTopology.NNConf("nn1").setIpcPort(10001))
.addNN(new MiniDFSNNTopology.NNConf("nn2").setIpcPort(10002))
.addNN(new MiniDFSNNTopology.NNConf("nn3").setIpcPort(10003)));
// add the configurations of the NNs to the passed conf, so we can parse it back out
MiniDFSCluster.configureNameNodes(topology, false, conf);
// set the 'local' one as nn1
conf.set(DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY, "nn1");
List<RemoteNameNodeInfo> nns = RemoteNameNodeInfo.getRemoteNameNodes(conf);
// make sure it matches when we pass in the nameservice
List<RemoteNameNodeInfo> nns2 = RemoteNameNodeInfo.getRemoteNameNodes(conf,
nameservice);
assertEquals(nns, nns2);
}
}

179
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestSeveralNameNodes.java Normal file
View File

@ -0,0 +1,179 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode.ha;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.test.MultithreadedTestUtil.RepeatingTestThread;
import org.apache.hadoop.test.MultithreadedTestUtil.TestContext;
import org.junit.Test;
/**
* Test that we can start several and run with namenodes on the same minicluster
*/
public class TestSeveralNameNodes {
private static final Log LOG = LogFactory.getLog(TestSeveralNameNodes.class);
/** ms between failovers between NNs */
private static final int TIME_BETWEEN_FAILOVERS = 200;
private static final int NUM_NAMENODES = 3;
private static final int NUM_THREADS = 3;
private static final int LIST_LENGTH = 50;
/** ms for length of test */
private static final long RUNTIME = 100000;
@Test
public void testCircularLinkedListWrites() throws Exception {
HAStressTestHarness harness = new HAStressTestHarness();
// setup the harness
harness.setNumberOfNameNodes(NUM_NAMENODES);
harness.addFailoverThread(TIME_BETWEEN_FAILOVERS);
final MiniDFSCluster cluster = harness.startCluster();
try {
cluster.waitActive();
cluster.transitionToActive(0);
// setup the a circular writer
FileSystem fs = harness.getFailoverFs();
TestContext context = harness.testCtx;
List<CircularWriter> writers = new ArrayList<CircularWriter>();
for (int i = 0; i < NUM_THREADS; i++) {
Path p = new Path("/test-" + i);
fs.mkdirs(p);
CircularWriter writer = new CircularWriter(context, LIST_LENGTH, fs, p);
writers.add(writer);
context.addThread(writer);
}
harness.startThreads();
// wait for all the writer threads to finish, or that we exceed the time
long start = System.currentTimeMillis();
while ((System.currentTimeMillis() - start) < RUNTIME) {
for (int i = 0; i < writers.size(); i++) {
CircularWriter writer = writers.get(i);
// remove the writer from the ones to check
if (writer.done.await(10, TimeUnit.MILLISECONDS)) {
writers.remove(i--);
}
}
}
assertEquals(
"Some writers didn't complete in expected runtime! Current writer state:"
+ writers, 0,
writers.size());
harness.stopThreads();
} finally {
System.err.println("===========================\n\n\n\n");
harness.shutdown();
}
}
private static class CircularWriter extends RepeatingTestThread {
private final int maxLength;
private final Path dir;
private final FileSystem fs;
private int currentListIndex = 0;
private CountDownLatch done = new CountDownLatch(1);
public CircularWriter(TestContext context, int listLength, FileSystem fs,
Path parentDir) {
super(context);
this.fs = fs;
this.maxLength = listLength;
this.dir = parentDir;
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder("Circular Writer:\n");
builder.append("\t directory: " + dir + "\n");
builder.append("\t target length: " + maxLength + "\n");
// might be a little racy, but we just want a close count
builder.append("\t current item: " + currentListIndex + "\n");
builder.append("\t done: " + (done.getCount() == 0) + "\n");
return builder.toString();
}
@Override
public void doAnAction() throws Exception {
if (currentListIndex == maxLength) {
checkList();
this.stopTestThread();
done.countDown();
} else {
writeList();
}
}
/**
* Make sure we can traverse the entire linked list
*/
private void checkList() throws IOException {
for (int i = 0; i < maxLength; i++) {
Path nextFile = getNextFile(i);
if (!fs.exists(nextFile)) {
throw new RuntimeException("Next file " + nextFile
+ " for list does not exist!");
}
// read the next file name
FSDataInputStream in = fs.open(nextFile);
nextFile = getNextFile(in.read());
in.close();
}
}
private void cleanup() throws IOException {
if (!fs.delete(dir, true)) {
throw new RuntimeException("Didn't correctly delete " + dir);
}
if (!fs.mkdirs(dir)) {
throw new RuntimeException("Didn't correctly make directory " + dir);
}
}
private void writeList() throws IOException {
Path nextPath = getNextFile(currentListIndex++);
LOG.info("Writing next file: " + nextPath);
FSDataOutputStream file = fs.create(nextPath);
file.write(currentListIndex);
file.close();
}
private Path getNextFile(int i) {
return new Path(dir, Integer.toString(i));
}
}
}

106
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java
View File

@ -63,8 +63,9 @@ import static org.junit.Assert.*;
public class TestStandbyCheckpoints {
private static final int NUM_DIRS_IN_LOG = 200000;
protected static int NUM_NNS = 3;
protected MiniDFSCluster cluster;
protected NameNode nn0, nn1;
protected NameNode[] nns = new NameNode[NUM_NNS];
protected FileSystem fs;
private final Random random = new Random();
protected File tmpOivImgDir;
@ -88,7 +89,8 @@ public class TestStandbyCheckpoints {
MiniDFSNNTopology topology = new MiniDFSNNTopology()
.addNameservice(new MiniDFSNNTopology.NSConf("ns1")
.addNN(new MiniDFSNNTopology.NNConf("nn1").setHttpPort(basePort))
.addNN(new MiniDFSNNTopology.NNConf("nn2").setHttpPort(basePort + 1)));
.addNN(new MiniDFSNNTopology.NNConf("nn2").setHttpPort(basePort + 1))
.addNN(new MiniDFSNNTopology.NNConf("nn3").setHttpPort(basePort + 2)));
cluster = new MiniDFSCluster.Builder(conf)
.nnTopology(topology)
@ -96,8 +98,8 @@ public class TestStandbyCheckpoints {
.build();
cluster.waitActive();
nn0 = cluster.getNameNode(0);
nn1 = cluster.getNameNode(1);
setNNs();
fs = HATestUtil.configureFailoverFs(cluster, conf);
cluster.transitionToActive(0);
@ -110,6 +112,12 @@ public class TestStandbyCheckpoints {
}
}
protected void setNNs(){
for (int i = 0; i < NUM_NNS; i++) {
nns[i] = cluster.getNameNode(i);
}
}
protected Configuration setupCommonConfig() {
tmpOivImgDir = Files.createTempDir();
@ -136,10 +144,10 @@ public class TestStandbyCheckpoints {
@Test(timeout = 300000)
public void testSBNCheckpoints() throws Exception {
JournalSet standbyJournalSet = NameNodeAdapter.spyOnJournalSet(nn1);
JournalSet standbyJournalSet = NameNodeAdapter.spyOnJournalSet(nns[1]);
doEdits(0, 10);
HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
HATestUtil.waitForStandbyToCatchUp(nns[0], nns[1]);
// Once the standby catches up, it should notice that it needs to
// do a checkpoint and save one to its local directories.
HATestUtil.waitForCheckpoint(cluster, 1, ImmutableList.of(12));
@ -147,10 +155,9 @@ public class TestStandbyCheckpoints {
GenericTestUtils.waitFor(new Supplier<Boolean>() {
@Override
public Boolean get() {
if(tmpOivImgDir.list().length > 0) {
if (tmpOivImgDir.list().length > 0) {
return true;
}
else {
} else {
return false;
}
}
@ -189,9 +196,9 @@ public class TestStandbyCheckpoints {
HATestUtil.waitForCheckpoint(cluster, 1, ImmutableList.of(12));
HATestUtil.waitForCheckpoint(cluster, 0, ImmutableList.of(12));
assertEquals(12, nn0.getNamesystem().getFSImage()
assertEquals(12, nns[0].getNamesystem().getFSImage()
.getMostRecentCheckpointTxId());
assertEquals(12, nn1.getNamesystem().getFSImage()
assertEquals(12, nns[1].getNamesystem().getFSImage()
.getMostRecentCheckpointTxId());
List<File> dirs = Lists.newArrayList();
@ -214,17 +221,17 @@ public class TestStandbyCheckpoints {
cluster.getConfiguration(1).setInt(
DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY, 0);
cluster.restartNameNode(1);
nn1 = cluster.getNameNode(1);
FSImage spyImage1 = NameNodeAdapter.spyOnFsImage(nn1);
nns[1] = cluster.getNameNode(1);
FSImage spyImage1 = NameNodeAdapter.spyOnFsImage(nns[1]);
// We shouldn't save any checkpoints at txid=0
Thread.sleep(1000);
Mockito.verify(spyImage1, Mockito.never())
.saveNamespace((FSNamesystem) Mockito.anyObject());
// Roll the primary and wait for the standby to catch up
HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
HATestUtil.waitForStandbyToCatchUp(nns[0], nns[1]);
Thread.sleep(2000);
// We should make exactly one checkpoint at this new txid.
@ -259,7 +266,7 @@ public class TestStandbyCheckpoints {
cluster.getConfiguration(1).setInt(
DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY, 0);
cluster.restartNameNode(1);
nn1 = cluster.getNameNode(1);
nns[1] = cluster.getNameNode(1);
cluster.transitionToActive(0);
@ -284,31 +291,42 @@ public class TestStandbyCheckpoints {
@Test(timeout=60000)
public void testCheckpointCancellationDuringUpload() throws Exception {
// don't compress, we want a big image
cluster.getConfiguration(0).setBoolean(
DFSConfigKeys.DFS_IMAGE_COMPRESS_KEY, false);
cluster.getConfiguration(1).setBoolean(
DFSConfigKeys.DFS_IMAGE_COMPRESS_KEY, false);
for (int i = 0; i < NUM_NNS; i++) {
cluster.getConfiguration(i).setBoolean(
DFSConfigKeys.DFS_IMAGE_COMPRESS_KEY, false);
}
// Throttle SBN upload to make it hang during upload to ANN
cluster.getConfiguration(1).setLong(
DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_KEY, 100);
cluster.restartNameNode(0);
cluster.restartNameNode(1);
nn0 = cluster.getNameNode(0);
nn1 = cluster.getNameNode(1);
for (int i = 1; i < NUM_NNS; i++) {
cluster.getConfiguration(i).setLong(
DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_KEY, 100);
}
for (int i = 0; i < NUM_NNS; i++) {
cluster.restartNameNode(i);
}
// update references to each of the nns
setNNs();
cluster.transitionToActive(0);
doEdits(0, 100);
HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
HATestUtil.waitForCheckpoint(cluster, 1, ImmutableList.of(104));
for (int i = 1; i < NUM_NNS; i++) {
HATestUtil.waitForStandbyToCatchUp(nns[0], nns[i]);
HATestUtil.waitForCheckpoint(cluster, i, ImmutableList.of(104));
}
cluster.transitionToStandby(0);
cluster.transitionToActive(1);
// Wait to make sure background TransferFsImageUpload thread was cancelled.
// This needs to be done before the next test in the suite starts, so that a
// file descriptor is not held open during the next cluster init.
cluster.shutdown();
cluster = null;
GenericTestUtils.waitFor(new Supplier<Boolean>() {
@Override
public Boolean get() {
@ -325,7 +343,7 @@ public class TestStandbyCheckpoints {
}, 1000, 30000);
// Assert that former active did not accept the canceled checkpoint file.
assertEquals(0, nn0.getFSImage().getMostRecentCheckpointTxId());
assertEquals(0, nns[0].getFSImage().getMostRecentCheckpointTxId());
}
/**
@ -337,7 +355,7 @@ public class TestStandbyCheckpoints {
public void testStandbyExceptionThrownDuringCheckpoint() throws Exception {
// Set it up so that we know when the SBN checkpoint starts and ends.
FSImage spyImage1 = NameNodeAdapter.spyOnFsImage(nn1);
FSImage spyImage1 = NameNodeAdapter.spyOnFsImage(nns[1]);
DelayAnswer answerer = new DelayAnswer(LOG);
Mockito.doAnswer(answerer).when(spyImage1)
.saveNamespace(Mockito.any(FSNamesystem.class),
@ -345,7 +363,7 @@ public class TestStandbyCheckpoints {
// Perform some edits and wait for a checkpoint to start on the SBN.
doEdits(0, 1000);
nn0.getRpcServer().rollEditLog();
nns[0].getRpcServer().rollEditLog();
answerer.waitForCall();
assertTrue("SBN is not performing checkpoint but it should be.",
answerer.getFireCount() == 1 && answerer.getResultCount() == 0);
@ -355,7 +373,7 @@ public class TestStandbyCheckpoints {
ThreadUtil.sleepAtLeastIgnoreInterrupts(1000);
try {
// Perform an RPC to the SBN and make sure it throws a StandbyException.
nn1.getRpcServer().getFileInfo("/");
nns[1].getRpcServer().getFileInfo("/");
fail("Should have thrown StandbyException, but instead succeeded.");
} catch (StandbyException se) {
GenericTestUtils.assertExceptionContains("is not supported", se);
@ -382,7 +400,7 @@ public class TestStandbyCheckpoints {
public void testReadsAllowedDuringCheckpoint() throws Exception {
// Set it up so that we know when the SBN checkpoint starts and ends.
FSImage spyImage1 = NameNodeAdapter.spyOnFsImage(nn1);
FSImage spyImage1 = NameNodeAdapter.spyOnFsImage(nns[1]);
DelayAnswer answerer = new DelayAnswer(LOG);
Mockito.doAnswer(answerer).when(spyImage1)
.saveNamespace(Mockito.any(FSNamesystem.class),
@ -391,7 +409,7 @@ public class TestStandbyCheckpoints {
// Perform some edits and wait for a checkpoint to start on the SBN.
doEdits(0, 1000);
nn0.getRpcServer().rollEditLog();
nns[0].getRpcServer().rollEditLog();
answerer.waitForCall();
assertTrue("SBN is not performing checkpoint but it should be.",
answerer.getFireCount() == 1 && answerer.getResultCount() == 0);
@ -405,7 +423,7 @@ public class TestStandbyCheckpoints {
@Override
public void run() {
try {
nn1.getRpcServer().restoreFailedStorage("false");
nns[1].getRpcServer().restoreFailedStorage("false");
} catch (IOException e) {
e.printStackTrace();
}
@ -415,16 +433,16 @@ public class TestStandbyCheckpoints {
// Make sure that our thread is waiting for the lock.
ThreadUtil.sleepAtLeastIgnoreInterrupts(1000);
assertFalse(nn1.getNamesystem().getFsLockForTests().hasQueuedThreads());
assertFalse(nn1.getNamesystem().getFsLockForTests().isWriteLocked());
assertTrue(nn1.getNamesystem().getCpLockForTests().hasQueuedThreads());
assertFalse(nns[1].getNamesystem().getFsLockForTests().hasQueuedThreads());
assertFalse(nns[1].getNamesystem().getFsLockForTests().isWriteLocked());
assertTrue(nns[1].getNamesystem().getCpLockForTests().hasQueuedThreads());
// Get /jmx of the standby NN web UI, which will cause the FSNS read lock to
// be taken.
String pageContents = DFSTestUtil.urlGet(new URL("http://" +
nn1.getHttpAddress().getHostName() + ":" +
nn1.getHttpAddress().getPort() + "/jmx"));
nns[1].getHttpAddress().getHostName() + ":" +
nns[1].getHttpAddress().getPort() + "/jmx"));
assertTrue(pageContents.contains("NumLiveDataNodes"));
// Make sure that the checkpoint is still going on, implying that the client

262
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForXAttr.java Normal file
View File

@ -0,0 +1,262 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.tools.offlineImageViewer;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
import org.apache.hadoop.hdfs.web.JsonUtil;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.apache.hadoop.net.NetUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
/**
* Tests OfflineImageViewer if the input fsimage has XAttributes
*/
public class TestOfflineImageViewerForXAttr {
private static final Log LOG = LogFactory
.getLog(TestOfflineImageViewerForXAttr.class);
private static File originalFsimage = null;
static String attr1JSon;
/**
* Create a populated namespace for later testing. Save its contents to a data
* structure and store its fsimage location. We only want to generate the
* fsimage file once and use it for multiple tests.
*/
@BeforeClass
public static void createOriginalFSImage() throws IOException {
MiniDFSCluster cluster = null;
Configuration conf = new Configuration();
try {
cluster = new MiniDFSCluster.Builder(conf).build();
cluster.waitActive();
DistributedFileSystem hdfs = cluster.getFileSystem();
// Create a name space with XAttributes
Path dir = new Path("/dir1");
hdfs.mkdirs(dir);
hdfs.setXAttr(dir, "user.attr1", "value1".getBytes());
hdfs.setXAttr(dir, "user.attr2", "value2".getBytes());
// Write results to the fsimage file
hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER, false);
hdfs.saveNamespace();
List<XAttr> attributes = new ArrayList<XAttr>();
attributes.add(XAttrHelper.buildXAttr("user.attr1", "value1".getBytes()));
attr1JSon = JsonUtil.toJsonString(attributes, null);
attributes.add(XAttrHelper.buildXAttr("user.attr2", "value2".getBytes()));
// Determine the location of the fsimage file
originalFsimage = FSImageTestUtil.findLatestImageFile(FSImageTestUtil
.getFSImage(cluster.getNameNode()).getStorage().getStorageDir(0));
if (originalFsimage == null) {
throw new RuntimeException("Didn't generate or can't find fsimage");
}
LOG.debug("original FS image file is " + originalFsimage);
} finally {
if (cluster != null)
cluster.shutdown();
}
}
@AfterClass
public static void deleteOriginalFSImage() throws IOException {
if (originalFsimage != null && originalFsimage.exists()) {
originalFsimage.delete();
}
}
@Test
public void testWebImageViewerForListXAttrs() throws Exception {
try (WebImageViewer viewer = new WebImageViewer(
NetUtils.createSocketAddr("localhost:0"))) {
viewer.initServer(originalFsimage.getAbsolutePath());
int port = viewer.getPort();
URL url = new URL("http://localhost:" + port
+ "/webhdfs/v1/dir1/?op=LISTXATTRS");
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setRequestMethod("GET");
connection.connect();
assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
String content = IOUtils.toString(connection.getInputStream());
assertTrue("Missing user.attr1 in response ",
content.contains("user.attr1"));
assertTrue("Missing user.attr2 in response ",
content.contains("user.attr2"));
}
}
@Test
public void testWebImageViewerForGetXAttrsWithOutParameters()
throws Exception {
try (WebImageViewer viewer = new WebImageViewer(
NetUtils.createSocketAddr("localhost:0"))) {
viewer.initServer(originalFsimage.getAbsolutePath());
int port = viewer.getPort();
URL url = new URL("http://localhost:" + port
+ "/webhdfs/v1/dir1/?op=GETXATTRS");
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setRequestMethod("GET");
connection.connect();
assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
String content = IOUtils.toString(connection.getInputStream());
assertTrue("Missing user.attr1 in response ",
content.contains("user.attr1"));
assertTrue("Missing user.attr2 in response ",
content.contains("user.attr2"));
}
}
@Test
public void testWebImageViewerForGetXAttrsWithParameters() throws Exception {
try (WebImageViewer viewer = new WebImageViewer(
NetUtils.createSocketAddr("localhost:0"))) {
viewer.initServer(originalFsimage.getAbsolutePath());
int port = viewer.getPort();
URL url = new URL("http://localhost:" + port
+ "/webhdfs/v1/dir1/?op=GETXATTRS&xattr.name=attr8");
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setRequestMethod("GET");
connection.connect();
assertEquals(HttpURLConnection.HTTP_BAD_REQUEST,
connection.getResponseCode());
url = new URL("http://localhost:" + port
+ "/webhdfs/v1/dir1/?op=GETXATTRS&xattr.name=user.attr1");
connection = (HttpURLConnection) url.openConnection();
connection.setRequestMethod("GET");
connection.connect();
assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
String content = IOUtils.toString(connection.getInputStream());
assertEquals(attr1JSon, content);
}
}
@Test
public void testWebImageViewerForGetXAttrsWithCodecParameters()
throws Exception {
try (WebImageViewer viewer = new WebImageViewer(
NetUtils.createSocketAddr("localhost:0"))) {
viewer.initServer(originalFsimage.getAbsolutePath());
int port = viewer.getPort();
URL url = new URL(
"http://localhost:"
+ port
+ "/webhdfs/v1/dir1/?op=GETXATTRS&xattr.name=USER.attr1&encoding=TEXT");
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setRequestMethod("GET");
connection.connect();
assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
String content = IOUtils.toString(connection.getInputStream());
assertEquals(attr1JSon, content);
}
}
@Test
public void testWithWebHdfsFileSystem() throws Exception {
try (WebImageViewer viewer = new WebImageViewer(
NetUtils.createSocketAddr("localhost:0"))) {
viewer.initServer(originalFsimage.getAbsolutePath());
int port = viewer.getPort();
// create a WebHdfsFileSystem instance
URI uri = new URI("webhdfs://localhost:" + String.valueOf(port));
Configuration conf = new Configuration();
WebHdfsFileSystem webhdfs = (WebHdfsFileSystem) FileSystem.get(uri, conf);
List<String> names = webhdfs.listXAttrs(new Path("/dir1"));
assertTrue(names.contains("user.attr1"));
assertTrue(names.contains("user.attr2"));
String value = new String(webhdfs.getXAttr(new Path("/dir1"),
"user.attr1"));
assertEquals("value1", value);
Map<String, byte[]> contentMap = webhdfs.getXAttrs(new Path("/dir1"),
names);
assertEquals("value1", new String(contentMap.get("user.attr1")));
assertEquals("value2", new String(contentMap.get("user.attr2")));
}
}
@Test
public void testResponseCode() throws Exception {
try (WebImageViewer viewer = new WebImageViewer(
NetUtils.createSocketAddr("localhost:0"))) {
viewer.initServer(originalFsimage.getAbsolutePath());
int port = viewer.getPort();
URL url = new URL(
"http://localhost:"
+ port
+ "/webhdfs/v1/dir1/?op=GETXATTRS&xattr.name=user.notpresent&encoding=TEXT");
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setRequestMethod("GET");
connection.connect();
assertEquals(HttpURLConnection.HTTP_FORBIDDEN,
connection.getResponseCode());
}
}
}

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/test/MiniDFSClusterManager.java
View File

@ -65,6 +65,7 @@ public class MiniDFSClusterManager {
private String writeDetails;
private int numDataNodes;
private int nameNodePort;
private int nameNodeHttpPort;
private StartupOption dfsOpts;
private String writeConfig;
private Configuration conf;
@ -84,6 +85,7 @@ public class MiniDFSClusterManager {
.addOption("cmdport", true,
"Which port to listen on for commands (default 0--we choose)")
.addOption("nnport", true, "NameNode port (default 0--we choose)")
.addOption("httpport", true, "NameNode http port (default 0--we choose)")
.addOption("namenode", true, "URL of the namenode (default "
+ "is either the DFS cluster or a temporary dir)")
.addOption(OptionBuilder
@ -137,6 +139,7 @@ public class MiniDFSClusterManager {
*/
public void start() throws IOException, FileNotFoundException {
dfs = new MiniDFSCluster.Builder(conf).nameNodePort(nameNodePort)
.nameNodeHttpPort(nameNodeHttpPort)
.numDataNodes(numDataNodes)
.startupOption(dfsOpts)
.format(format)
@ -198,6 +201,7 @@ public class MiniDFSClusterManager {
// HDFS
numDataNodes = intArgument(cli, "datanodes", 1);
nameNodePort = intArgument(cli, "nnport", 0);
nameNodeHttpPort = intArgument(cli, "httpport", 0);
if (cli.hasOption("format")) {
dfsOpts = StartupOption.FORMAT;
format = true;

BIN
hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-0.23-reserved.tgz
View File

Binary file not shown.

BIN
hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-1-reserved.tgz
View File

Binary file not shown.

BIN
hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-2-reserved.tgz
View File

Binary file not shown.

BIN
hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-22-dfs-dir.tgz
View File

Binary file not shown.

BIN
hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop1-bbw.tgz
View File

Binary file not shown.

2
hadoop-hdfs-project/hadoop-hdfs/src/test/resources/log4j.properties
View File

@ -20,4 +20,4 @@ log4j.rootLogger=info,stdout
log4j.threshold=ALL
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n

10
hadoop-mapreduce-project/CHANGES.txt
View File

@ -505,6 +505,13 @@ Release 2.8.0 - UNRELEASED
multibyte record delimiters well (Vinayakumar B, Rushabh Shah, and Akira
AJISAKA via jlowe)
MAPREDUCE-6400. Multiple shuffle transfer fails because input is closed
too early (Brahma Reddy Battula, Akira AJISAKA, and Gera Shegalov via
jlowe)
MAPREDUCE-6413. TestLocalJobSubmission is failing with unknown host
(zhihai xu via jlowe)
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
@ -548,6 +555,9 @@ Release 2.7.1 - UNRELEASED
MAPREDUCE-6387. Serialize the recently added Task#encryptedSpillKey field at
the end. (Arun Suresh via kasha)
MAPREDUCE-6410. Fixed MapReduce JobHistory server to use the right (login)
UGI to refresh log and cleaner settings. (Varun Saxena via vinodkv)
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES

1
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
View File

@ -335,6 +335,7 @@ class Fetcher<K,V> extends Thread {
try {
failedTasks = copyMapOutput(host, input, remaining, fetchRetryEnabled);
} catch (IOException e) {
IOUtils.cleanup(LOG, input);
//
// Setup connection again if disconnected by NM
connection.disconnect();

10
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/IFileWrappedMapOutput.java
View File

@ -60,13 +60,7 @@ public abstract class IFileWrappedMapOutput<K, V> extends MapOutput<K, V> {
long compressedLength, long decompressedLength,
ShuffleClientMetrics metrics,
Reporter reporter) throws IOException {
IFileInputStream iFin =
new IFileInputStream(input, compressedLength, conf);
try {
this.doShuffle(host, iFin, compressedLength,
decompressedLength, metrics, reporter);
} finally {
iFin.close();
}
doShuffle(host, new IFileInputStream(input, compressedLength, conf),
compressedLength, decompressedLength, metrics, reporter);
}
}

15
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/LocalFetcher.java
View File

@ -30,6 +30,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.mapred.IndexRecord;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MapOutputFile;
@ -149,19 +150,13 @@ class LocalFetcher<K,V> extends Fetcher<K, V> {
// now read the file, seek to the appropriate section, and send it.
FileSystem localFs = FileSystem.getLocal(job).getRaw();
FSDataInputStream inStream = localFs.open(mapOutputFileName);
inStream = CryptoUtils.wrapIfNecessary(job, inStream);
try {
inStream = CryptoUtils.wrapIfNecessary(job, inStream);
inStream.seek(ir.startOffset + CryptoUtils.cryptoPadding(job));
mapOutput.shuffle(LOCALHOST, inStream, compressedLength, decompressedLength, metrics, reporter);
mapOutput.shuffle(LOCALHOST, inStream, compressedLength,
decompressedLength, metrics, reporter);
} finally {
try {
inStream.close();
} catch (IOException ioe) {
LOG.warn("IOException closing inputstream from map output: "
+ ioe.toString());
}
IOUtils.cleanup(LOG, inStream);
}
scheduler.copySucceeded(mapTaskId, LOCALHOST, compressedLength, 0, 0,

43
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
View File

@ -20,6 +20,7 @@ package org.apache.hadoop.mapreduce.v2.hs.server;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.security.PrivilegedExceptionAction;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -52,6 +53,7 @@ import org.apache.hadoop.mapreduce.v2.hs.JobHistory;
import org.apache.hadoop.mapreduce.v2.hs.proto.HSAdminRefreshProtocolProtos.HSAdminRefreshProtocolService;
import org.apache.hadoop.mapreduce.v2.hs.protocolPB.HSAdminRefreshProtocolServerSideTranslatorPB;
import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.BlockingService;
@Private
@ -67,6 +69,8 @@ public class HSAdminServer extends AbstractService implements HSAdminProtocol {
private static final String HISTORY_ADMIN_SERVER = "HSAdminServer";
private JobHistory jobHistoryService = null;
private UserGroupInformation loginUGI;
public HSAdminServer(AggregatedLogDeletionService aggLogDelService,
JobHistory jobHistoryService) {
super(HSAdminServer.class.getName());
@ -125,9 +129,24 @@ public class HSAdminServer extends AbstractService implements HSAdminProtocol {
@Override
protected void serviceStart() throws Exception {
if (UserGroupInformation.isSecurityEnabled()) {
loginUGI = UserGroupInformation.getLoginUser();
} else {
loginUGI = UserGroupInformation.getCurrentUser();
}
clientRpcServer.start();
}
@VisibleForTesting
UserGroupInformation getLoginUGI() {
return loginUGI;
}
@VisibleForTesting
void setLoginUGI(UserGroupInformation ugi) {
loginUGI = ugi;
}
@Override
protected void serviceStop() throws Exception {
if (clientRpcServer != null) {
@ -233,7 +252,17 @@ public class HSAdminServer extends AbstractService implements HSAdminProtocol {
public void refreshLogRetentionSettings() throws IOException {
UserGroupInformation user = checkAcls("refreshLogRetentionSettings");
aggLogDelService.refreshLogRetentionSettings();
try {
loginUGI.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws IOException {
aggLogDelService.refreshLogRetentionSettings();
return null;
}
});
} catch (InterruptedException e) {
throw new IOException(e);
}
HSAuditLogger.logSuccess(user.getShortUserName(),
"refreshLogRetentionSettings", "HSAdminServer");
@ -243,7 +272,17 @@ public class HSAdminServer extends AbstractService implements HSAdminProtocol {
public void refreshJobRetentionSettings() throws IOException {
UserGroupInformation user = checkAcls("refreshJobRetentionSettings");
jobHistoryService.refreshJobRetentionSettings();
try {
loginUGI.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws IOException {
jobHistoryService.refreshJobRetentionSettings();
return null;
}
});
} catch (InterruptedException e) {
throw new IOException(e);
}
HSAuditLogger.logSuccess(user.getShortUserName(),
"refreshJobRetentionSettings", HISTORY_ADMIN_SERVER);

55
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java
View File

@ -21,6 +21,8 @@ package org.apache.hadoop.mapreduce.v2.hs.server;
import static org.junit.Assert.*;
import java.io.IOException;
import java.security.PrivilegedAction;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@ -45,6 +47,9 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.mockito.Mockito.verify;
@ -286,6 +291,56 @@ public class TestHSAdminServer {
verify(jobHistoryService).refreshJobRetentionSettings();
}
@SuppressWarnings("unchecked")
@Test
public void testUGIForLogAndJobRefresh() throws Exception {
UserGroupInformation ugi =
UserGroupInformation.createUserForTesting("test", new String[] {"grp"});
UserGroupInformation loginUGI = spy(hsAdminServer.getLoginUGI());
hsAdminServer.setLoginUGI(loginUGI);
// Run refresh log retention settings with test user
ugi.doAs(new PrivilegedAction<Void>() {
@Override
public Void run() {
String[] args = new String[1];
args[0] = "-refreshLogRetentionSettings";
try {
hsAdminClient.run(args);
} catch (Exception e) {
fail("refreshLogRetentionSettings should have been successful");
}
return null;
}
});
// Verify if AggregatedLogDeletionService#refreshLogRetentionSettings was
// called with login UGI, instead of the UGI command was run with.
verify(loginUGI).doAs(any(PrivilegedExceptionAction.class));
verify(alds).refreshLogRetentionSettings();
// Reset for refresh job retention settings
reset(loginUGI);
// Run refresh job retention settings with test user
ugi.doAs(new PrivilegedAction<Void>() {
@Override
public Void run() {
String[] args = new String[1];
args[0] = "-refreshJobRetentionSettings";
try {
hsAdminClient.run(args);
} catch (Exception e) {
fail("refreshJobRetentionSettings should have been successful");
}
return null;
}
});
// Verify if JobHistory#refreshJobRetentionSettings was called with
// login UGI, instead of the UGI command was run with.
verify(loginUGI).doAs(any(PrivilegedExceptionAction.class));
verify(jobHistoryService).refreshJobRetentionSettings();
}
@After
public void cleanUp() {
if (hsAdminServer != null)

4
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLocalJobSubmission.java
View File

@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.SleepJob;
import org.apache.hadoop.util.ToolRunner;
import org.junit.After;
@ -63,7 +64,8 @@ public class TestLocalJobSubmission {
Path jarPath = makeJar(new Path(TEST_ROOT_DIR, "test.jar"));
Configuration conf = new Configuration();
conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "hdfs://testcluster");
conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "hdfs://localhost:9000");
conf.set(MRConfig.FRAMEWORK_NAME, "local");
final String[] args = {
"-jt" , "local", "-libjars", jarPath.toString(),
"-m", "1", "-r", "1", "-mt", "1", "-rt", "1"

2
hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
View File

@ -30,7 +30,7 @@ public class DistCpConstants {
public static final int DEFAULT_MAPS = 20;
/* Default bandwidth if none specified */
public static final int DEFAULT_BANDWIDTH_MB = 100;
public static final float DEFAULT_BANDWIDTH_MB = 100;
/* Default strategy for copying. Implementation looked up
from distcp-default.xml

5
hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
View File

@ -174,10 +174,11 @@ public enum DistCpOptionSwitch {
"copied to <= n bytes")),
/**
* Specify bandwidth per map in MB
* Specify bandwidth per map in MB, accepts bandwidth as a fraction
*/
BANDWIDTH(DistCpConstants.CONF_LABEL_BANDWIDTH_MB,
new Option("bandwidth", true, "Specify bandwidth per map in MB")),
new Option("bandwidth", true, "Specify bandwidth per map in MB,"
+ " accepts bandwidth as a fraction.")),
/**
* Path containing a list of strings, which when found in the path of

Some files were not shown because too many files have changed in this diff Show More