Merge r1609845 through r1619277 from trunk.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-6584@1619293 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
commit
c92d869d02
21
BUILDING.txt
21
BUILDING.txt
|
@ -81,6 +81,27 @@ Maven build goals:
|
||||||
the final tar file. This option requires that -Dsnappy.lib is also given,
|
the final tar file. This option requires that -Dsnappy.lib is also given,
|
||||||
and it ignores the -Dsnappy.prefix option.
|
and it ignores the -Dsnappy.prefix option.
|
||||||
|
|
||||||
|
OpenSSL build options:
|
||||||
|
|
||||||
|
OpenSSL includes a crypto library that can be utilized by the native code.
|
||||||
|
It is currently an optional component, meaning that Hadoop can be built with
|
||||||
|
or without this dependency.
|
||||||
|
|
||||||
|
* Use -Drequire.openssl to fail the build if libcrypto.so is not found.
|
||||||
|
If this option is not specified and the openssl library is missing,
|
||||||
|
we silently build a version of libhadoop.so that cannot make use of
|
||||||
|
openssl. This option is recommended if you plan on making use of openssl
|
||||||
|
and want to get more repeatable builds.
|
||||||
|
* Use -Dopenssl.prefix to specify a nonstandard location for the libcrypto
|
||||||
|
header files and library files. You do not need this option if you have
|
||||||
|
installed openssl using a package manager.
|
||||||
|
* Use -Dopenssl.lib to specify a nonstandard location for the libcrypto library
|
||||||
|
files. Similarly to openssl.prefix, you do not need this option if you have
|
||||||
|
installed openssl using a package manager.
|
||||||
|
* Use -Dbundle.openssl to copy the contents of the openssl.lib directory into
|
||||||
|
the final tar file. This option requires that -Dopenssl.lib is also given,
|
||||||
|
and it ignores the -Dopenssl.prefix option.
|
||||||
|
|
||||||
Tests options:
|
Tests options:
|
||||||
|
|
||||||
* Use -DskipTests to skip tests when running the following Maven goals:
|
* Use -DskipTests to skip tests when running the following Maven goals:
|
||||||
|
|
|
@ -29,6 +29,7 @@
|
||||||
<exclude>*-config.cmd</exclude>
|
<exclude>*-config.cmd</exclude>
|
||||||
<exclude>start-*.cmd</exclude>
|
<exclude>start-*.cmd</exclude>
|
||||||
<exclude>stop-*.cmd</exclude>
|
<exclude>stop-*.cmd</exclude>
|
||||||
|
<exclude>hadoop-layout.sh.example</exclude>
|
||||||
</excludes>
|
</excludes>
|
||||||
<fileMode>0755</fileMode>
|
<fileMode>0755</fileMode>
|
||||||
</fileSet>
|
</fileSet>
|
||||||
|
@ -42,6 +43,8 @@
|
||||||
<includes>
|
<includes>
|
||||||
<include>*-config.sh</include>
|
<include>*-config.sh</include>
|
||||||
<include>*-config.cmd</include>
|
<include>*-config.cmd</include>
|
||||||
|
<include>*-functions.sh</include>
|
||||||
|
<include>hadoop-layout.sh.example</include>
|
||||||
</includes>
|
</includes>
|
||||||
<fileMode>0755</fileMode>
|
<fileMode>0755</fileMode>
|
||||||
</fileSet>
|
</fileSet>
|
||||||
|
@ -57,6 +60,10 @@
|
||||||
<exclude>hadoop.cmd</exclude>
|
<exclude>hadoop.cmd</exclude>
|
||||||
<exclude>hdfs.cmd</exclude>
|
<exclude>hdfs.cmd</exclude>
|
||||||
<exclude>hadoop-config.cmd</exclude>
|
<exclude>hadoop-config.cmd</exclude>
|
||||||
|
<exclude>hadoop-functions.sh</exclude>
|
||||||
|
<exclude>hadoop-layout.sh.example</exclude>
|
||||||
|
<exclude>hdfs-config.cmd</exclude>
|
||||||
|
<exclude>hdfs-config.sh</exclude>
|
||||||
</excludes>
|
</excludes>
|
||||||
<fileMode>0755</fileMode>
|
<fileMode>0755</fileMode>
|
||||||
</fileSet>
|
</fileSet>
|
||||||
|
|
|
@ -9,6 +9,8 @@ Trunk (Unreleased)
|
||||||
|
|
||||||
HADOOP-10474 Move o.a.h.record to hadoop-streaming. (wheat9)
|
HADOOP-10474 Move o.a.h.record to hadoop-streaming. (wheat9)
|
||||||
|
|
||||||
|
HADOOP-9902. Shell script rewrite (aw)
|
||||||
|
|
||||||
NEW FEATURES
|
NEW FEATURES
|
||||||
|
|
||||||
HADOOP-10433. Key Management Server based on KeyProvider API. (tucu)
|
HADOOP-10433. Key Management Server based on KeyProvider API. (tucu)
|
||||||
|
@ -440,6 +442,56 @@ Trunk (Unreleased)
|
||||||
|
|
||||||
HADOOP-8589. ViewFs tests fail when tests and home dirs are nested (sanjay Radia)
|
HADOOP-8589. ViewFs tests fail when tests and home dirs are nested (sanjay Radia)
|
||||||
|
|
||||||
|
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
|
||||||
|
|
||||||
|
HADOOP-10734. Implement high-performance secure random number sources.
|
||||||
|
(Yi Liu via Colin Patrick McCabe)
|
||||||
|
|
||||||
|
HADOOP-10603. Crypto input and output streams implementing Hadoop stream
|
||||||
|
interfaces. (Yi Liu and Charles Lamb)
|
||||||
|
|
||||||
|
HADOOP-10628. Javadoc and few code style improvement for Crypto
|
||||||
|
input and output streams. (Yi Liu via clamb)
|
||||||
|
|
||||||
|
HADOOP-10632. Minor improvements to Crypto input and output streams.
|
||||||
|
(Yi Liu)
|
||||||
|
|
||||||
|
HADOOP-10635. Add a method to CryptoCodec to generate SRNs for IV. (Yi Liu)
|
||||||
|
|
||||||
|
HADOOP-10653. Add a new constructor for CryptoInputStream that
|
||||||
|
receives current position of wrapped stream. (Yi Liu)
|
||||||
|
|
||||||
|
HADOOP-10662. NullPointerException in CryptoInputStream while wrapped
|
||||||
|
stream is not ByteBufferReadable. Add tests using normal stream. (Yi Liu)
|
||||||
|
|
||||||
|
HADOOP-10713. Refactor CryptoCodec#generateSecureRandom to take a byte[].
|
||||||
|
(wang via yliu)
|
||||||
|
|
||||||
|
HADOOP-10693. Implementation of AES-CTR CryptoCodec using JNI to OpenSSL.
|
||||||
|
(Yi Liu via cmccabe)
|
||||||
|
|
||||||
|
HADOOP-10803. Update OpensslCipher#getInstance to accept CipherSuite#name
|
||||||
|
format. (Yi Liu)
|
||||||
|
|
||||||
|
HADOOP-10735. Fall back AesCtrCryptoCodec implementation from OpenSSL to
|
||||||
|
JCE if non native support. (Yi Liu)
|
||||||
|
|
||||||
|
HADOOP-10870. Failed to load OpenSSL cipher error logs on systems with old
|
||||||
|
openssl versions (cmccabe)
|
||||||
|
|
||||||
|
HADOOP-10853. Refactor get instance of CryptoCodec and support create via
|
||||||
|
algorithm/mode/padding. (Yi Liu)
|
||||||
|
|
||||||
|
HADOOP-10919. Copy command should preserve raw.* namespace
|
||||||
|
extended attributes. (clamb)
|
||||||
|
|
||||||
|
HDFS-6873. Constants in CommandWithDestination should be static. (clamb)
|
||||||
|
|
||||||
|
HADOOP-10871. incorrect prototype in OpensslSecureRandom.c (cmccabe)
|
||||||
|
|
||||||
|
HADOOP-10886. CryptoCodec#getCodecclasses throws NPE when configurations not
|
||||||
|
loaded. (umamahesh)
|
||||||
|
|
||||||
Release 2.6.0 - UNRELEASED
|
Release 2.6.0 - UNRELEASED
|
||||||
|
|
||||||
INCOMPATIBLE CHANGES
|
INCOMPATIBLE CHANGES
|
||||||
|
@ -621,7 +673,10 @@ Release 2.6.0 - UNRELEASED
|
||||||
HADOOP-10873. Fix dead link in Configuration javadoc (Akira AJISAKA
|
HADOOP-10873. Fix dead link in Configuration javadoc (Akira AJISAKA
|
||||||
via aw)
|
via aw)
|
||||||
|
|
||||||
Release 2.5.0 - UNRELEASED
|
HADOOP-10968. hadoop native build fails to detect java_libarch on
|
||||||
|
ppc64le (Dinar Valeev via Colin Patrick McCabe)
|
||||||
|
|
||||||
|
Release 2.5.0 - 2014-08-11
|
||||||
|
|
||||||
INCOMPATIBLE CHANGES
|
INCOMPATIBLE CHANGES
|
||||||
|
|
||||||
|
|
|
@ -499,6 +499,10 @@
|
||||||
<snappy.lib></snappy.lib>
|
<snappy.lib></snappy.lib>
|
||||||
<snappy.include></snappy.include>
|
<snappy.include></snappy.include>
|
||||||
<require.snappy>false</require.snappy>
|
<require.snappy>false</require.snappy>
|
||||||
|
<openssl.prefix></openssl.prefix>
|
||||||
|
<openssl.lib></openssl.lib>
|
||||||
|
<openssl.include></openssl.include>
|
||||||
|
<require.openssl>false</require.openssl>
|
||||||
</properties>
|
</properties>
|
||||||
<build>
|
<build>
|
||||||
<plugins>
|
<plugins>
|
||||||
|
@ -548,6 +552,8 @@
|
||||||
<javahClassName>org.apache.hadoop.io.compress.snappy.SnappyDecompressor</javahClassName>
|
<javahClassName>org.apache.hadoop.io.compress.snappy.SnappyDecompressor</javahClassName>
|
||||||
<javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Compressor</javahClassName>
|
<javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Compressor</javahClassName>
|
||||||
<javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Decompressor</javahClassName>
|
<javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Decompressor</javahClassName>
|
||||||
|
<javahClassName>org.apache.hadoop.crypto.OpensslCipher</javahClassName>
|
||||||
|
<javahClassName>org.apache.hadoop.crypto.random.OpensslSecureRandom</javahClassName>
|
||||||
<javahClassName>org.apache.hadoop.util.NativeCrc32</javahClassName>
|
<javahClassName>org.apache.hadoop.util.NativeCrc32</javahClassName>
|
||||||
<javahClassName>org.apache.hadoop.net.unix.DomainSocket</javahClassName>
|
<javahClassName>org.apache.hadoop.net.unix.DomainSocket</javahClassName>
|
||||||
<javahClassName>org.apache.hadoop.net.unix.DomainSocketWatcher</javahClassName>
|
<javahClassName>org.apache.hadoop.net.unix.DomainSocketWatcher</javahClassName>
|
||||||
|
@ -568,7 +574,7 @@
|
||||||
<configuration>
|
<configuration>
|
||||||
<target>
|
<target>
|
||||||
<exec executable="cmake" dir="${project.build.directory}/native" failonerror="true">
|
<exec executable="cmake" dir="${project.build.directory}/native" failonerror="true">
|
||||||
<arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_BZIP2=${require.bzip2} -DREQUIRE_SNAPPY=${require.snappy} -DCUSTOM_SNAPPY_PREFIX=${snappy.prefix} -DCUSTOM_SNAPPY_LIB=${snappy.lib} -DCUSTOM_SNAPPY_INCLUDE=${snappy.include}"/>
|
<arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_BZIP2=${require.bzip2} -DREQUIRE_SNAPPY=${require.snappy} -DCUSTOM_SNAPPY_PREFIX=${snappy.prefix} -DCUSTOM_SNAPPY_LIB=${snappy.lib} -DCUSTOM_SNAPPY_INCLUDE=${snappy.include} -DREQUIRE_OPENSSL=${require.openssl} -DCUSTOM_OPENSSL_PREFIX=${openssl.prefix} -DCUSTOM_OPENSSL_LIB=${openssl.lib} -DCUSTOM_OPENSSL_INCLUDE=${openssl.include}"/>
|
||||||
</exec>
|
</exec>
|
||||||
<exec executable="make" dir="${project.build.directory}/native" failonerror="true">
|
<exec executable="make" dir="${project.build.directory}/native" failonerror="true">
|
||||||
<arg line="VERBOSE=1"/>
|
<arg line="VERBOSE=1"/>
|
||||||
|
@ -612,6 +618,11 @@
|
||||||
<snappy.include></snappy.include>
|
<snappy.include></snappy.include>
|
||||||
<require.snappy>false</require.snappy>
|
<require.snappy>false</require.snappy>
|
||||||
<bundle.snappy.in.bin>true</bundle.snappy.in.bin>
|
<bundle.snappy.in.bin>true</bundle.snappy.in.bin>
|
||||||
|
<openssl.prefix></openssl.prefix>
|
||||||
|
<openssl.lib></openssl.lib>
|
||||||
|
<openssl.include></openssl.include>
|
||||||
|
<require.openssl>false</require.openssl>
|
||||||
|
<bundle.openssl.in.bin>true</bundle.openssl.in.bin>
|
||||||
</properties>
|
</properties>
|
||||||
<build>
|
<build>
|
||||||
<plugins>
|
<plugins>
|
||||||
|
@ -657,6 +668,8 @@
|
||||||
<javahClassName>org.apache.hadoop.io.compress.snappy.SnappyDecompressor</javahClassName>
|
<javahClassName>org.apache.hadoop.io.compress.snappy.SnappyDecompressor</javahClassName>
|
||||||
<javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Compressor</javahClassName>
|
<javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Compressor</javahClassName>
|
||||||
<javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Decompressor</javahClassName>
|
<javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Decompressor</javahClassName>
|
||||||
|
<javahClassName>org.apache.hadoop.crypto.OpensslCipher</javahClassName>
|
||||||
|
<javahClassName>org.apache.hadoop.crypto.random.OpensslSecureRandom</javahClassName>
|
||||||
<javahClassName>org.apache.hadoop.util.NativeCrc32</javahClassName>
|
<javahClassName>org.apache.hadoop.util.NativeCrc32</javahClassName>
|
||||||
</javahClassNames>
|
</javahClassNames>
|
||||||
<javahOutputDirectory>${project.build.directory}/native/javah</javahOutputDirectory>
|
<javahOutputDirectory>${project.build.directory}/native/javah</javahOutputDirectory>
|
||||||
|
@ -701,6 +714,10 @@
|
||||||
<argument>/p:CustomSnappyLib=${snappy.lib}</argument>
|
<argument>/p:CustomSnappyLib=${snappy.lib}</argument>
|
||||||
<argument>/p:CustomSnappyInclude=${snappy.include}</argument>
|
<argument>/p:CustomSnappyInclude=${snappy.include}</argument>
|
||||||
<argument>/p:RequireSnappy=${require.snappy}</argument>
|
<argument>/p:RequireSnappy=${require.snappy}</argument>
|
||||||
|
<argument>/p:CustomOpensslPrefix=${openssl.prefix}</argument>
|
||||||
|
<argument>/p:CustomOpensslLib=${openssl.lib}</argument>
|
||||||
|
<argument>/p:CustomOpensslInclude=${openssl.include}</argument>
|
||||||
|
<argument>/p:RequireOpenssl=${require.openssl}</argument>
|
||||||
</arguments>
|
</arguments>
|
||||||
</configuration>
|
</configuration>
|
||||||
</execution>
|
</execution>
|
||||||
|
|
|
@ -145,6 +145,38 @@ else (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
|
||||||
ENDIF(REQUIRE_SNAPPY)
|
ENDIF(REQUIRE_SNAPPY)
|
||||||
endif (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
|
endif (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
|
||||||
|
|
||||||
|
SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
|
||||||
|
set_find_shared_library_version("1.0.0")
|
||||||
|
SET(OPENSSL_NAME "crypto")
|
||||||
|
IF(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
|
||||||
|
SET(OPENSSL_NAME "eay32")
|
||||||
|
ENDIF()
|
||||||
|
find_library(OPENSSL_LIBRARY
|
||||||
|
NAMES ${OPENSSL_NAME}
|
||||||
|
PATHS ${CUSTOM_OPENSSL_PREFIX} ${CUSTOM_OPENSSL_PREFIX}/lib
|
||||||
|
${CUSTOM_OPENSSL_PREFIX}/lib64 ${CUSTOM_OPENSSL_LIB} NO_DEFAULT_PATH)
|
||||||
|
find_library(OPENSSL_LIBRARY
|
||||||
|
NAMES ${OPENSSL_NAME})
|
||||||
|
SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES)
|
||||||
|
find_path(OPENSSL_INCLUDE_DIR
|
||||||
|
NAMES openssl/evp.h
|
||||||
|
PATHS ${CUSTOM_OPENSSL_PREFIX} ${CUSTOM_OPENSSL_PREFIX}/include
|
||||||
|
${CUSTOM_OPENSSL_INCLUDE} NO_DEFAULT_PATH)
|
||||||
|
find_path(OPENSSL_INCLUDE_DIR
|
||||||
|
NAMES openssl/evp.h)
|
||||||
|
if (OPENSSL_LIBRARY AND OPENSSL_INCLUDE_DIR)
|
||||||
|
GET_FILENAME_COMPONENT(HADOOP_OPENSSL_LIBRARY ${OPENSSL_LIBRARY} NAME)
|
||||||
|
SET(OPENSSL_SOURCE_FILES
|
||||||
|
"${D}/crypto/OpensslCipher.c"
|
||||||
|
"${D}/crypto/random/OpensslSecureRandom.c")
|
||||||
|
else (OPENSSL_LIBRARY AND OPENSSL_INCLUDE_DIR)
|
||||||
|
SET(OPENSSL_INCLUDE_DIR "")
|
||||||
|
SET(OPENSSL_SOURCE_FILES "")
|
||||||
|
IF(REQUIRE_OPENSSL)
|
||||||
|
MESSAGE(FATAL_ERROR "Required openssl library could not be found. OPENSSL_LIBRARY=${OPENSSL_LIBRARY}, OPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}, CUSTOM_OPENSSL_INCLUDE_DIR=${CUSTOM_OPENSSL_INCLUDE_DIR}, CUSTOM_OPENSSL_PREFIX=${CUSTOM_OPENSSL_PREFIX}, CUSTOM_OPENSSL_INCLUDE=${CUSTOM_OPENSSL_INCLUDE}")
|
||||||
|
ENDIF(REQUIRE_OPENSSL)
|
||||||
|
endif (OPENSSL_LIBRARY AND OPENSSL_INCLUDE_DIR)
|
||||||
|
|
||||||
include_directories(
|
include_directories(
|
||||||
${GENERATED_JAVAH}
|
${GENERATED_JAVAH}
|
||||||
main/native/src
|
main/native/src
|
||||||
|
@ -155,6 +187,7 @@ include_directories(
|
||||||
${ZLIB_INCLUDE_DIRS}
|
${ZLIB_INCLUDE_DIRS}
|
||||||
${BZIP2_INCLUDE_DIR}
|
${BZIP2_INCLUDE_DIR}
|
||||||
${SNAPPY_INCLUDE_DIR}
|
${SNAPPY_INCLUDE_DIR}
|
||||||
|
${OPENSSL_INCLUDE_DIR}
|
||||||
${D}/util
|
${D}/util
|
||||||
)
|
)
|
||||||
CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
|
CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
|
||||||
|
@ -172,6 +205,7 @@ add_dual_library(hadoop
|
||||||
${D}/io/compress/lz4/lz4.c
|
${D}/io/compress/lz4/lz4.c
|
||||||
${D}/io/compress/lz4/lz4hc.c
|
${D}/io/compress/lz4/lz4hc.c
|
||||||
${SNAPPY_SOURCE_FILES}
|
${SNAPPY_SOURCE_FILES}
|
||||||
|
${OPENSSL_SOURCE_FILES}
|
||||||
${D}/io/compress/zlib/ZlibCompressor.c
|
${D}/io/compress/zlib/ZlibCompressor.c
|
||||||
${D}/io/compress/zlib/ZlibDecompressor.c
|
${D}/io/compress/zlib/ZlibDecompressor.c
|
||||||
${BZIP2_SOURCE_FILES}
|
${BZIP2_SOURCE_FILES}
|
||||||
|
|
|
@ -78,6 +78,12 @@ IF("${CMAKE_SYSTEM}" MATCHES "Linux")
|
||||||
SET(_java_libarch "amd64")
|
SET(_java_libarch "amd64")
|
||||||
ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "^arm")
|
ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "^arm")
|
||||||
SET(_java_libarch "arm")
|
SET(_java_libarch "arm")
|
||||||
|
ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64le")
|
||||||
|
IF(EXISTS "${_JAVA_HOME}/jre/lib/ppc64le")
|
||||||
|
SET(_java_libarch "ppc64le")
|
||||||
|
ELSE()
|
||||||
|
SET(_java_libarch "ppc64")
|
||||||
|
ENDIF()
|
||||||
ELSE()
|
ELSE()
|
||||||
SET(_java_libarch ${CMAKE_SYSTEM_PROCESSOR})
|
SET(_java_libarch ${CMAKE_SYSTEM_PROCESSOR})
|
||||||
ENDIF()
|
ENDIF()
|
||||||
|
|
|
@ -21,6 +21,7 @@
|
||||||
#cmakedefine HADOOP_ZLIB_LIBRARY "@HADOOP_ZLIB_LIBRARY@"
|
#cmakedefine HADOOP_ZLIB_LIBRARY "@HADOOP_ZLIB_LIBRARY@"
|
||||||
#cmakedefine HADOOP_BZIP2_LIBRARY "@HADOOP_BZIP2_LIBRARY@"
|
#cmakedefine HADOOP_BZIP2_LIBRARY "@HADOOP_BZIP2_LIBRARY@"
|
||||||
#cmakedefine HADOOP_SNAPPY_LIBRARY "@HADOOP_SNAPPY_LIBRARY@"
|
#cmakedefine HADOOP_SNAPPY_LIBRARY "@HADOOP_SNAPPY_LIBRARY@"
|
||||||
|
#cmakedefine HADOOP_OPENSSL_LIBRARY "@HADOOP_OPENSSL_LIBRARY@"
|
||||||
#cmakedefine HAVE_SYNC_FILE_RANGE
|
#cmakedefine HAVE_SYNC_FILE_RANGE
|
||||||
#cmakedefine HAVE_POSIX_FADVISE
|
#cmakedefine HAVE_POSIX_FADVISE
|
||||||
|
|
||||||
|
|
|
@ -15,130 +15,164 @@
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
# This script runs the hadoop core commands.
|
function hadoop_usage()
|
||||||
|
{
|
||||||
bin=`which $0`
|
|
||||||
bin=`dirname ${bin}`
|
|
||||||
bin=`cd "$bin" > /dev/null; pwd`
|
|
||||||
|
|
||||||
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
|
|
||||||
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
|
|
||||||
. $HADOOP_LIBEXEC_DIR/hadoop-config.sh
|
|
||||||
|
|
||||||
function print_usage(){
|
|
||||||
echo "Usage: hadoop [--config confdir] COMMAND"
|
echo "Usage: hadoop [--config confdir] COMMAND"
|
||||||
echo " where COMMAND is one of:"
|
echo " where COMMAND is one of:"
|
||||||
echo " fs run a generic filesystem user client"
|
echo " archive -archiveName NAME -p <parent path> <src>* <dest>"
|
||||||
echo " version print the version"
|
echo " create a Hadoop archive"
|
||||||
echo " jar <jar> run a jar file"
|
echo " checknative [-a|-h] check native Hadoop and compression "
|
||||||
echo " checknative [-a|-h] check native hadoop and compression libraries availability"
|
echo " libraries availability"
|
||||||
echo " distcp <srcurl> <desturl> copy file or directories recursively"
|
|
||||||
echo " archive -archiveName NAME -p <parent path> <src>* <dest> create a hadoop archive"
|
|
||||||
echo " classpath prints the class path needed to get the"
|
echo " classpath prints the class path needed to get the"
|
||||||
|
echo " Hadoop jar and the required libraries"
|
||||||
echo " credential interact with credential providers"
|
echo " credential interact with credential providers"
|
||||||
echo " Hadoop jar and the required libraries"
|
|
||||||
echo " daemonlog get/set the log level for each daemon"
|
echo " daemonlog get/set the log level for each daemon"
|
||||||
|
echo " distch path:owner:group:permisson"
|
||||||
|
echo " distributed metadata changer"
|
||||||
|
echo " distcp <srcurl> <desturl> "
|
||||||
|
echo " copy file or directories recursively"
|
||||||
|
echo " fs run a generic filesystem user client"
|
||||||
|
echo " jar <jar> run a jar file"
|
||||||
|
echo " jnipath prints the java.library.path"
|
||||||
|
echo " key manage keys via the KeyProvider"
|
||||||
|
echo " version print the version"
|
||||||
echo " or"
|
echo " or"
|
||||||
echo " CLASSNAME run the class named CLASSNAME"
|
echo " CLASSNAME run the class named CLASSNAME"
|
||||||
echo ""
|
echo ""
|
||||||
echo "Most commands print help when invoked w/o parameters."
|
echo "Most commands print help when invoked w/o parameters."
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# This script runs the hadoop core commands.
|
||||||
|
|
||||||
|
# let's locate libexec...
|
||||||
|
if [[ -n "${HADOOP_PREFIX}" ]]; then
|
||||||
|
DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
|
||||||
|
else
|
||||||
|
this="${BASH_SOURCE-$0}"
|
||||||
|
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
|
||||||
|
DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
|
||||||
|
fi
|
||||||
|
|
||||||
|
HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
HADOOP_NEW_CONFIG=true
|
||||||
|
if [[ -f "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
|
||||||
|
. "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
|
||||||
|
else
|
||||||
|
echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh." 2>&1
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
if [ $# = 0 ]; then
|
if [ $# = 0 ]; then
|
||||||
print_usage
|
hadoop_exit_with_usage 1
|
||||||
exit
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
COMMAND=$1
|
COMMAND=$1
|
||||||
case $COMMAND in
|
shift
|
||||||
# usage flags
|
|
||||||
--help|-help|-h)
|
|
||||||
print_usage
|
|
||||||
exit
|
|
||||||
;;
|
|
||||||
|
|
||||||
#hdfs commands
|
case ${COMMAND} in
|
||||||
namenode|secondarynamenode|datanode|dfs|dfsadmin|fsck|balancer|fetchdt|oiv|dfsgroups|portmap|nfs3)
|
balancer|datanode|dfs|dfsadmin|dfsgroups| \
|
||||||
echo "DEPRECATED: Use of this script to execute hdfs command is deprecated." 1>&2
|
namenode|secondarynamenode|fsck|fetchdt|oiv| \
|
||||||
echo "Instead use the hdfs command for it." 1>&2
|
portmap|nfs3)
|
||||||
echo "" 1>&2
|
hadoop_error "WARNING: Use of this script to execute ${COMMAND} is deprecated."
|
||||||
#try to locate hdfs and if present, delegate to it.
|
COMMAND=${COMMAND/dfsgroups/groups}
|
||||||
shift
|
hadoop_error "WARNING: Attempting to execute replacement \"hdfs ${COMMAND}\" instead."
|
||||||
if [ -f "${HADOOP_HDFS_HOME}"/bin/hdfs ]; then
|
hadoop_error ""
|
||||||
exec "${HADOOP_HDFS_HOME}"/bin/hdfs ${COMMAND/dfsgroups/groups} "$@"
|
#try to locate hdfs and if present, delegate to it.
|
||||||
elif [ -f "${HADOOP_PREFIX}"/bin/hdfs ]; then
|
if [[ -f "${HADOOP_HDFS_HOME}/bin/hdfs" ]]; then
|
||||||
exec "${HADOOP_PREFIX}"/bin/hdfs ${COMMAND/dfsgroups/groups} "$@"
|
# shellcheck disable=SC2086
|
||||||
|
exec "${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||||
|
--config "${HADOOP_CONF_DIR}" "${COMMAND}" "$@"
|
||||||
|
elif [[ -f "${HADOOP_PREFIX}/bin/hdfs" ]]; then
|
||||||
|
# shellcheck disable=SC2086
|
||||||
|
exec "${HADOOP_PREFIX}/bin/hdfs" \
|
||||||
|
--config "${HADOOP_CONF_DIR}" "${COMMAND}" "$@"
|
||||||
else
|
else
|
||||||
echo "HADOOP_HDFS_HOME not found!"
|
hadoop_error "HADOOP_HDFS_HOME not found!"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
;;
|
;;
|
||||||
|
|
||||||
#mapred commands for backwards compatibility
|
#mapred commands for backwards compatibility
|
||||||
pipes|job|queue|mrgroups|mradmin|jobtracker|tasktracker)
|
pipes|job|queue|mrgroups|mradmin|jobtracker|tasktracker)
|
||||||
echo "DEPRECATED: Use of this script to execute mapred command is deprecated." 1>&2
|
hadoop_error "WARNING: Use of this script to execute ${COMMAND} is deprecated."
|
||||||
echo "Instead use the mapred command for it." 1>&2
|
COMMAND=${COMMAND/mrgroups/groups}
|
||||||
echo "" 1>&2
|
hadoop_error "WARNING: Attempting to execute replacement \"mapred ${COMMAND}\" instead."
|
||||||
|
hadoop_error ""
|
||||||
#try to locate mapred and if present, delegate to it.
|
#try to locate mapred and if present, delegate to it.
|
||||||
shift
|
if [[ -f "${HADOOP_MAPRED_HOME}/bin/mapred" ]]; then
|
||||||
if [ -f "${HADOOP_MAPRED_HOME}"/bin/mapred ]; then
|
exec "${HADOOP_MAPRED_HOME}/bin/mapred" \
|
||||||
exec "${HADOOP_MAPRED_HOME}"/bin/mapred ${COMMAND/mrgroups/groups} "$@"
|
--config "${HADOOP_CONF_DIR}" "${COMMAND}" "$@"
|
||||||
elif [ -f "${HADOOP_PREFIX}"/bin/mapred ]; then
|
elif [[ -f "${HADOOP_PREFIX}/bin/mapred" ]]; then
|
||||||
exec "${HADOOP_PREFIX}"/bin/mapred ${COMMAND/mrgroups/groups} "$@"
|
exec "${HADOOP_PREFIX}/bin/mapred" \
|
||||||
|
--config "${HADOOP_CONF_DIR}" "${COMMAND}" "$@"
|
||||||
else
|
else
|
||||||
echo "HADOOP_MAPRED_HOME not found!"
|
hadoop_error "HADOOP_MAPRED_HOME not found!"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
;;
|
;;
|
||||||
|
archive)
|
||||||
#core commands
|
CLASS=org.apache.hadoop.tools.HadoopArchives
|
||||||
*)
|
hadoop_add_classpath "${TOOL_PATH}"
|
||||||
# the core commands
|
;;
|
||||||
if [ "$COMMAND" = "fs" ] ; then
|
checknative)
|
||||||
CLASS=org.apache.hadoop.fs.FsShell
|
CLASS=org.apache.hadoop.util.NativeLibraryChecker
|
||||||
elif [ "$COMMAND" = "version" ] ; then
|
;;
|
||||||
CLASS=org.apache.hadoop.util.VersionInfo
|
classpath)
|
||||||
elif [ "$COMMAND" = "jar" ] ; then
|
if [[ "$#" -eq 1 ]]; then
|
||||||
CLASS=org.apache.hadoop.util.RunJar
|
CLASS=org.apache.hadoop.util.Classpath
|
||||||
elif [ "$COMMAND" = "key" ] ; then
|
else
|
||||||
CLASS=org.apache.hadoop.crypto.key.KeyShell
|
hadoop_finalize
|
||||||
elif [ "$COMMAND" = "checknative" ] ; then
|
echo "${CLASSPATH}"
|
||||||
CLASS=org.apache.hadoop.util.NativeLibraryChecker
|
exit 0
|
||||||
elif [ "$COMMAND" = "distcp" ] ; then
|
|
||||||
CLASS=org.apache.hadoop.tools.DistCp
|
|
||||||
CLASSPATH=${CLASSPATH}:${TOOL_PATH}
|
|
||||||
elif [ "$COMMAND" = "daemonlog" ] ; then
|
|
||||||
CLASS=org.apache.hadoop.log.LogLevel
|
|
||||||
elif [ "$COMMAND" = "archive" ] ; then
|
|
||||||
CLASS=org.apache.hadoop.tools.HadoopArchives
|
|
||||||
CLASSPATH=${CLASSPATH}:${TOOL_PATH}
|
|
||||||
elif [ "$COMMAND" = "credential" ] ; then
|
|
||||||
CLASS=org.apache.hadoop.security.alias.CredentialShell
|
|
||||||
elif [ "$COMMAND" = "classpath" ] ; then
|
|
||||||
if [ "$#" -eq 1 ]; then
|
|
||||||
# No need to bother starting up a JVM for this simple case.
|
|
||||||
echo $CLASSPATH
|
|
||||||
exit
|
|
||||||
else
|
|
||||||
CLASS=org.apache.hadoop.util.Classpath
|
|
||||||
fi
|
|
||||||
elif [[ "$COMMAND" = -* ]] ; then
|
|
||||||
# class and package names cannot begin with a -
|
|
||||||
echo "Error: No command named \`$COMMAND' was found. Perhaps you meant \`hadoop ${COMMAND#-}'"
|
|
||||||
exit 1
|
|
||||||
else
|
|
||||||
CLASS=$COMMAND
|
|
||||||
fi
|
fi
|
||||||
shift
|
;;
|
||||||
|
credential)
|
||||||
# Always respect HADOOP_OPTS and HADOOP_CLIENT_OPTS
|
CLASS=org.apache.hadoop.security.alias.CredentialShell
|
||||||
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
|
;;
|
||||||
|
daemonlog)
|
||||||
#make sure security appender is turned off
|
CLASS=org.apache.hadoop.log.LogLevel
|
||||||
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,NullAppender}"
|
;;
|
||||||
|
distch)
|
||||||
export CLASSPATH=$CLASSPATH
|
CLASS=org.apache.hadoop.tools.DistCh
|
||||||
exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS $CLASS "$@"
|
hadoop_add_classpath "${TOOL_PATH}"
|
||||||
;;
|
;;
|
||||||
|
distcp)
|
||||||
|
CLASS=org.apache.hadoop.tools.DistCp
|
||||||
|
hadoop_add_classpath "${TOOL_PATH}"
|
||||||
|
;;
|
||||||
|
fs)
|
||||||
|
CLASS=org.apache.hadoop.fs.FsShell
|
||||||
|
;;
|
||||||
|
jar)
|
||||||
|
CLASS=org.apache.hadoop.util.RunJar
|
||||||
|
;;
|
||||||
|
jnipath)
|
||||||
|
hadoop_finalize
|
||||||
|
echo "${JAVA_LIBRARY_PATH}"
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
key)
|
||||||
|
CLASS=org.apache.hadoop.crypto.key.KeyShell
|
||||||
|
;;
|
||||||
|
version)
|
||||||
|
CLASS=org.apache.hadoop.util.VersionInfo
|
||||||
|
;;
|
||||||
|
-*|hdfs)
|
||||||
|
hadoop_exit_with_usage 1
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
CLASS="${COMMAND}"
|
||||||
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
|
# Always respect HADOOP_OPTS and HADOOP_CLIENT_OPTS
|
||||||
|
HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
|
||||||
|
|
||||||
|
hadoop_add_param HADOOP_OPTS Xmx "${JAVA_HEAP_MAX}"
|
||||||
|
|
||||||
|
hadoop_finalize
|
||||||
|
export CLASSPATH
|
||||||
|
hadoop_java_exec "${COMMAND}" "${CLASS}" "$@"
|
||||||
|
|
||||||
|
|
|
@ -1,3 +1,5 @@
|
||||||
|
#
|
||||||
|
#
|
||||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
# contributor license agreements. See the NOTICE file distributed with
|
# contributor license agreements. See the NOTICE file distributed with
|
||||||
# this work for additional information regarding copyright ownership.
|
# this work for additional information regarding copyright ownership.
|
||||||
|
@ -13,280 +15,176 @@
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
|
####
|
||||||
|
# IMPORTANT
|
||||||
|
####
|
||||||
|
|
||||||
|
## The hadoop-config.sh tends to get executed by non-Hadoop scripts.
|
||||||
|
## Those parts expect this script to parse/manipulate $@. In order
|
||||||
|
## to maintain backward compatibility, this means a surprising
|
||||||
|
## lack of functions for bits that would be much better off in
|
||||||
|
## a function.
|
||||||
|
##
|
||||||
|
## In other words, yes, there is some bad things happen here and
|
||||||
|
## unless we break the rest of the ecosystem, we can't change it. :(
|
||||||
|
|
||||||
|
|
||||||
# included in all the hadoop scripts with source command
|
# included in all the hadoop scripts with source command
|
||||||
# should not be executable directly
|
# should not be executable directly
|
||||||
# also should not be passed any arguments, since we need original $*
|
# also should not be passed any arguments, since we need original $*
|
||||||
|
|
||||||
# Resolve links ($0 may be a softlink) and convert a relative path
|
|
||||||
# to an absolute path. NB: The -P option requires bash built-ins
|
|
||||||
# or POSIX:2001 compliant cd and pwd.
|
|
||||||
|
|
||||||
# HADOOP_CLASSPATH Extra Java CLASSPATH entries.
|
|
||||||
#
|
|
||||||
# HADOOP_USER_CLASSPATH_FIRST When defined, the HADOOP_CLASSPATH is
|
|
||||||
# added in the beginning of the global
|
|
||||||
# classpath. Can be defined, for example,
|
|
||||||
# by doing
|
|
||||||
# export HADOOP_USER_CLASSPATH_FIRST=true
|
|
||||||
#
|
#
|
||||||
|
# after doing more config, caller should also exec finalize
|
||||||
|
# function to finish last minute/default configs for
|
||||||
|
# settings that might be different between daemons & interactive
|
||||||
|
|
||||||
this="${BASH_SOURCE-$0}"
|
# you must be this high to ride the ride
|
||||||
common_bin=$(cd -P -- "$(dirname -- "$this")" && pwd -P)
|
if [[ -z "${BASH_VERSINFO}" ]] || [[ "${BASH_VERSINFO}" -lt 3 ]]; then
|
||||||
script="$(basename -- "$this")"
|
echo "Hadoop requires bash v3 or better. Sorry."
|
||||||
this="$common_bin/$script"
|
exit 1
|
||||||
|
|
||||||
[ -f "$common_bin/hadoop-layout.sh" ] && . "$common_bin/hadoop-layout.sh"
|
|
||||||
|
|
||||||
HADOOP_COMMON_DIR=${HADOOP_COMMON_DIR:-"share/hadoop/common"}
|
|
||||||
HADOOP_COMMON_LIB_JARS_DIR=${HADOOP_COMMON_LIB_JARS_DIR:-"share/hadoop/common/lib"}
|
|
||||||
HADOOP_COMMON_LIB_NATIVE_DIR=${HADOOP_COMMON_LIB_NATIVE_DIR:-"lib/native"}
|
|
||||||
HDFS_DIR=${HDFS_DIR:-"share/hadoop/hdfs"}
|
|
||||||
HDFS_LIB_JARS_DIR=${HDFS_LIB_JARS_DIR:-"share/hadoop/hdfs/lib"}
|
|
||||||
YARN_DIR=${YARN_DIR:-"share/hadoop/yarn"}
|
|
||||||
YARN_LIB_JARS_DIR=${YARN_LIB_JARS_DIR:-"share/hadoop/yarn/lib"}
|
|
||||||
MAPRED_DIR=${MAPRED_DIR:-"share/hadoop/mapreduce"}
|
|
||||||
MAPRED_LIB_JARS_DIR=${MAPRED_LIB_JARS_DIR:-"share/hadoop/mapreduce/lib"}
|
|
||||||
|
|
||||||
# the root of the Hadoop installation
|
|
||||||
# See HADOOP-6255 for directory structure layout
|
|
||||||
HADOOP_DEFAULT_PREFIX=$(cd -P -- "$common_bin"/.. && pwd -P)
|
|
||||||
HADOOP_PREFIX=${HADOOP_PREFIX:-$HADOOP_DEFAULT_PREFIX}
|
|
||||||
export HADOOP_PREFIX
|
|
||||||
|
|
||||||
#check to see if the conf dir is given as an optional argument
|
|
||||||
if [ $# -gt 1 ]
|
|
||||||
then
|
|
||||||
if [ "--config" = "$1" ]
|
|
||||||
then
|
|
||||||
shift
|
|
||||||
confdir=$1
|
|
||||||
if [ ! -d "$confdir" ]; then
|
|
||||||
echo "Error: Cannot find configuration directory: $confdir"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
shift
|
|
||||||
HADOOP_CONF_DIR=$confdir
|
|
||||||
fi
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Allow alternate conf dir location.
|
# In order to get partially bootstrapped, we need to figure out where
|
||||||
if [ -e "${HADOOP_PREFIX}/conf/hadoop-env.sh" ]; then
|
# we are located. Chances are good that our caller has already done
|
||||||
DEFAULT_CONF_DIR="conf"
|
# this work for us, but just in case...
|
||||||
|
|
||||||
|
if [[ -z "${HADOOP_LIBEXEC_DIR}" ]]; then
|
||||||
|
_hadoop_common_this="${BASH_SOURCE-$0}"
|
||||||
|
HADOOP_LIBEXEC_DIR=$(cd -P -- "$(dirname -- "${_hadoop_common_this}")" >/dev/null && pwd -P)
|
||||||
|
fi
|
||||||
|
|
||||||
|
# get our functions defined for usage later
|
||||||
|
if [[ -f "${HADOOP_LIBEXEC_DIR}/hadoop-functions.sh" ]]; then
|
||||||
|
. "${HADOOP_LIBEXEC_DIR}/hadoop-functions.sh"
|
||||||
else
|
else
|
||||||
DEFAULT_CONF_DIR="etc/hadoop"
|
echo "ERROR: Unable to exec ${HADOOP_LIBEXEC_DIR}/hadoop-functions.sh." 1>&2
|
||||||
fi
|
|
||||||
|
|
||||||
export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_PREFIX/$DEFAULT_CONF_DIR}"
|
|
||||||
|
|
||||||
if [ -f "${HADOOP_CONF_DIR}/hadoop-env.sh" ]; then
|
|
||||||
. "${HADOOP_CONF_DIR}/hadoop-env.sh"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# User can specify hostnames or a file where the hostnames are (not both)
|
|
||||||
if [[ ( "$HADOOP_SLAVES" != '' ) && ( "$HADOOP_SLAVE_NAMES" != '' ) ]] ; then
|
|
||||||
echo \
|
|
||||||
"Error: Please specify one variable HADOOP_SLAVES or " \
|
|
||||||
"HADOOP_SLAVE_NAME and not both."
|
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Process command line options that specify hosts or file with host
|
# allow overrides of the above and pre-defines of the below
|
||||||
# list
|
if [[ -f "${HADOOP_LIBEXEC_DIR}/hadoop-layout.sh" ]]; then
|
||||||
if [ $# -gt 1 ]
|
. "${HADOOP_LIBEXEC_DIR}/hadoop-layout.sh"
|
||||||
then
|
|
||||||
if [ "--hosts" = "$1" ]
|
|
||||||
then
|
|
||||||
shift
|
|
||||||
export HADOOP_SLAVES="${HADOOP_CONF_DIR}/$1"
|
|
||||||
shift
|
|
||||||
elif [ "--hostnames" = "$1" ]
|
|
||||||
then
|
|
||||||
shift
|
|
||||||
export HADOOP_SLAVE_NAMES=$1
|
|
||||||
shift
|
|
||||||
fi
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# User can specify hostnames or a file where the hostnames are (not both)
|
#
|
||||||
# (same check as above but now we know it's command line options that cause
|
# IMPORTANT! We are not executing user provided code yet!
|
||||||
# the problem)
|
#
|
||||||
if [[ ( "$HADOOP_SLAVES" != '' ) && ( "$HADOOP_SLAVE_NAMES" != '' ) ]] ; then
|
|
||||||
echo \
|
# Let's go! Base definitions so we can move forward
|
||||||
"Error: Please specify one of --hosts or --hostnames options and not both."
|
hadoop_bootstrap_init
|
||||||
exit 1
|
|
||||||
|
# let's find our conf.
|
||||||
|
#
|
||||||
|
# first, check and process params passed to us
|
||||||
|
# we process this in-line so that we can directly modify $@
|
||||||
|
# if something downstream is processing that directly,
|
||||||
|
# we need to make sure our params have been ripped out
|
||||||
|
# note that we do many of them here for various utilities.
|
||||||
|
# this provides consistency and forces a more consistent
|
||||||
|
# user experience
|
||||||
|
|
||||||
|
|
||||||
|
# save these off in case our caller needs them
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
HADOOP_USER_PARAMS="$@"
|
||||||
|
|
||||||
|
HADOOP_DAEMON_MODE="default"
|
||||||
|
|
||||||
|
while [[ -z "${_hadoop_common_done}" ]]; do
|
||||||
|
case $1 in
|
||||||
|
--buildpaths)
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
HADOOP_ENABLE_BUILD_PATHS=true
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
--config)
|
||||||
|
shift
|
||||||
|
confdir=$1
|
||||||
|
shift
|
||||||
|
if [[ -d "${confdir}" ]]; then
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
YARN_CONF_DIR="${confdir}"
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
HADOOP_CONF_DIR="${confdir}"
|
||||||
|
elif [[ -z "${confdir}" ]]; then
|
||||||
|
hadoop_error "ERROR: No parameter provided for --config "
|
||||||
|
hadoop_exit_with_usage 1
|
||||||
|
else
|
||||||
|
hadoop_error "ERROR: Cannot find configuration directory \"${confdir}\""
|
||||||
|
hadoop_exit_with_usage 1
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
--daemon)
|
||||||
|
shift
|
||||||
|
HADOOP_DAEMON_MODE=$1
|
||||||
|
shift
|
||||||
|
if [[ -z "${HADOOP_DAEMON_MODE}" || \
|
||||||
|
! "${HADOOP_DAEMON_MODE}" =~ ^st(art|op|atus)$ ]]; then
|
||||||
|
hadoop_error "ERROR: --daemon must be followed by either \"start\", \"stop\", or \"status\"."
|
||||||
|
hadoop_exit_with_usage 1
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
--help|-help|-h|help|--h|--\?|-\?|\?)
|
||||||
|
hadoop_exit_with_usage 0
|
||||||
|
;;
|
||||||
|
--hostnames)
|
||||||
|
shift
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
HADOOP_SLAVE_NAMES="$1"
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
--hosts)
|
||||||
|
shift
|
||||||
|
hadoop_populate_slaves_file "$1"
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
_hadoop_common_done=true
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
hadoop_find_confdir
|
||||||
|
hadoop_exec_hadoopenv
|
||||||
|
|
||||||
|
#
|
||||||
|
# IMPORTANT! User provided code is now available!
|
||||||
|
#
|
||||||
|
|
||||||
|
# do all the OS-specific startup bits here
|
||||||
|
# this allows us to get a decent JAVA_HOME,
|
||||||
|
# call crle for LD_LIBRARY_PATH, etc.
|
||||||
|
hadoop_os_tricks
|
||||||
|
|
||||||
|
hadoop_java_setup
|
||||||
|
|
||||||
|
hadoop_basic_init
|
||||||
|
|
||||||
|
# inject any sub-project overrides, defaults, etc.
|
||||||
|
if declare -F hadoop_subproject_init >/dev/null ; then
|
||||||
|
hadoop_subproject_init
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# check if net.ipv6.bindv6only is set to 1
|
# get the native libs in there pretty quick
|
||||||
bindv6only=$(/sbin/sysctl -n net.ipv6.bindv6only 2> /dev/null)
|
hadoop_add_javalibpath "${HADOOP_PREFIX}/build/native"
|
||||||
if [ -n "$bindv6only" ] && [ "$bindv6only" -eq "1" ] && [ "$HADOOP_ALLOW_IPV6" != "yes" ]
|
hadoop_add_javalibpath "${HADOOP_PREFIX}/${HADOOP_COMMON_LIB_NATIVE_DIR}"
|
||||||
then
|
|
||||||
echo "Error: \"net.ipv6.bindv6only\" is set to 1 - Java networking could be broken"
|
# get the basic java class path for these subprojects
|
||||||
echo "For more info: http://wiki.apache.org/hadoop/HadoopIPv6"
|
# in as quickly as possible since other stuff
|
||||||
exit 1
|
# will definitely depend upon it.
|
||||||
fi
|
#
|
||||||
|
# at some point, this will get replaced with something pluggable
|
||||||
# Newer versions of glibc use an arena memory allocator that causes virtual
|
# so that these functions can sit in their projects rather than
|
||||||
# memory usage to explode. This interacts badly with the many threads that
|
# common
|
||||||
# we use in Hadoop. Tune the variable down to prevent vmem explosion.
|
#
|
||||||
export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-4}
|
for i in common hdfs yarn mapred
|
||||||
|
do
|
||||||
# Attempt to set JAVA_HOME if it is not set
|
hadoop_add_to_classpath_$i
|
||||||
if [[ -z $JAVA_HOME ]]; then
|
done
|
||||||
# On OSX use java_home (or /Library for older versions)
|
|
||||||
if [ "Darwin" == "$(uname -s)" ]; then
|
#
|
||||||
if [ -x /usr/libexec/java_home ]; then
|
# backwards compatibility. new stuff should
|
||||||
export JAVA_HOME=($(/usr/libexec/java_home))
|
# call this when they are ready
|
||||||
else
|
#
|
||||||
export JAVA_HOME=(/Library/Java/Home)
|
if [[ -z "${HADOOP_NEW_CONFIG}" ]]; then
|
||||||
fi
|
hadoop_finalize
|
||||||
fi
|
|
||||||
|
|
||||||
# Bail if we did not detect it
|
|
||||||
if [[ -z $JAVA_HOME ]]; then
|
|
||||||
echo "Error: JAVA_HOME is not set and could not be found." 1>&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
JAVA=$JAVA_HOME/bin/java
|
|
||||||
|
|
||||||
# check envvars which might override default args
|
|
||||||
if [ "$HADOOP_HEAPSIZE" != "" ]; then
|
|
||||||
#echo "run with heapsize $HADOOP_HEAPSIZE"
|
|
||||||
JAVA_HEAP_MAX="-Xmx""$HADOOP_HEAPSIZE""m"
|
|
||||||
#echo $JAVA_HEAP_MAX
|
|
||||||
fi
|
|
||||||
|
|
||||||
# CLASSPATH initially contains $HADOOP_CONF_DIR
|
|
||||||
CLASSPATH="${HADOOP_CONF_DIR}"
|
|
||||||
|
|
||||||
# so that filenames w/ spaces are handled correctly in loops below
|
|
||||||
IFS=
|
|
||||||
|
|
||||||
if [ "$HADOOP_COMMON_HOME" = "" ]; then
|
|
||||||
if [ -d "${HADOOP_PREFIX}/$HADOOP_COMMON_DIR" ]; then
|
|
||||||
export HADOOP_COMMON_HOME=$HADOOP_PREFIX
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# for releases, add core hadoop jar & webapps to CLASSPATH
|
|
||||||
if [ -d "$HADOOP_COMMON_HOME/$HADOOP_COMMON_DIR/webapps" ]; then
|
|
||||||
CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/$HADOOP_COMMON_DIR
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -d "$HADOOP_COMMON_HOME/$HADOOP_COMMON_LIB_JARS_DIR" ]; then
|
|
||||||
CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/$HADOOP_COMMON_LIB_JARS_DIR'/*'
|
|
||||||
fi
|
|
||||||
|
|
||||||
CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/$HADOOP_COMMON_DIR'/*'
|
|
||||||
|
|
||||||
# default log directory & file
|
|
||||||
if [ "$HADOOP_LOG_DIR" = "" ]; then
|
|
||||||
HADOOP_LOG_DIR="$HADOOP_PREFIX/logs"
|
|
||||||
fi
|
|
||||||
if [ "$HADOOP_LOGFILE" = "" ]; then
|
|
||||||
HADOOP_LOGFILE='hadoop.log'
|
|
||||||
fi
|
|
||||||
|
|
||||||
# default policy file for service-level authorization
|
|
||||||
if [ "$HADOOP_POLICYFILE" = "" ]; then
|
|
||||||
HADOOP_POLICYFILE="hadoop-policy.xml"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# restore ordinary behaviour
|
|
||||||
unset IFS
|
|
||||||
|
|
||||||
# setup 'java.library.path' for native-hadoop code if necessary
|
|
||||||
|
|
||||||
if [ -d "${HADOOP_PREFIX}/build/native" -o -d "${HADOOP_PREFIX}/$HADOOP_COMMON_LIB_NATIVE_DIR" ]; then
|
|
||||||
|
|
||||||
if [ -d "${HADOOP_PREFIX}/$HADOOP_COMMON_LIB_NATIVE_DIR" ]; then
|
|
||||||
if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
|
|
||||||
JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:${HADOOP_PREFIX}/$HADOOP_COMMON_LIB_NATIVE_DIR
|
|
||||||
else
|
|
||||||
JAVA_LIBRARY_PATH=${HADOOP_PREFIX}/$HADOOP_COMMON_LIB_NATIVE_DIR
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# setup a default TOOL_PATH
|
|
||||||
TOOL_PATH="${TOOL_PATH:-$HADOOP_PREFIX/share/hadoop/tools/lib/*}"
|
|
||||||
|
|
||||||
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR"
|
|
||||||
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.file=$HADOOP_LOGFILE"
|
|
||||||
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.home.dir=$HADOOP_PREFIX"
|
|
||||||
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.id.str=$HADOOP_IDENT_STRING"
|
|
||||||
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.root.logger=${HADOOP_ROOT_LOGGER:-INFO,console}"
|
|
||||||
if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
|
|
||||||
HADOOP_OPTS="$HADOOP_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
|
|
||||||
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$JAVA_LIBRARY_PATH
|
|
||||||
fi
|
|
||||||
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.policy.file=$HADOOP_POLICYFILE"
|
|
||||||
|
|
||||||
# Disable ipv6 as it can cause issues
|
|
||||||
HADOOP_OPTS="$HADOOP_OPTS -Djava.net.preferIPv4Stack=true"
|
|
||||||
|
|
||||||
# put hdfs in classpath if present
|
|
||||||
if [ "$HADOOP_HDFS_HOME" = "" ]; then
|
|
||||||
if [ -d "${HADOOP_PREFIX}/$HDFS_DIR" ]; then
|
|
||||||
export HADOOP_HDFS_HOME=$HADOOP_PREFIX
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -d "$HADOOP_HDFS_HOME/$HDFS_DIR/webapps" ]; then
|
|
||||||
CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/$HDFS_DIR
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -d "$HADOOP_HDFS_HOME/$HDFS_LIB_JARS_DIR" ]; then
|
|
||||||
CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/$HDFS_LIB_JARS_DIR'/*'
|
|
||||||
fi
|
|
||||||
|
|
||||||
CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/$HDFS_DIR'/*'
|
|
||||||
|
|
||||||
# put yarn in classpath if present
|
|
||||||
if [ "$HADOOP_YARN_HOME" = "" ]; then
|
|
||||||
if [ -d "${HADOOP_PREFIX}/$YARN_DIR" ]; then
|
|
||||||
export HADOOP_YARN_HOME=$HADOOP_PREFIX
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -d "$HADOOP_YARN_HOME/$YARN_DIR/webapps" ]; then
|
|
||||||
CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/$YARN_DIR
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -d "$HADOOP_YARN_HOME/$YARN_LIB_JARS_DIR" ]; then
|
|
||||||
CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/$YARN_LIB_JARS_DIR'/*'
|
|
||||||
fi
|
|
||||||
|
|
||||||
CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/$YARN_DIR'/*'
|
|
||||||
|
|
||||||
# put mapred in classpath if present AND different from YARN
|
|
||||||
if [ "$HADOOP_MAPRED_HOME" = "" ]; then
|
|
||||||
if [ -d "${HADOOP_PREFIX}/$MAPRED_DIR" ]; then
|
|
||||||
export HADOOP_MAPRED_HOME=$HADOOP_PREFIX
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ "$HADOOP_MAPRED_HOME/$MAPRED_DIR" != "$HADOOP_YARN_HOME/$YARN_DIR" ] ; then
|
|
||||||
if [ -d "$HADOOP_MAPRED_HOME/$MAPRED_DIR/webapps" ]; then
|
|
||||||
CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/$MAPRED_DIR
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -d "$HADOOP_MAPRED_HOME/$MAPRED_LIB_JARS_DIR" ]; then
|
|
||||||
CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/$MAPRED_LIB_JARS_DIR'/*'
|
|
||||||
fi
|
|
||||||
|
|
||||||
CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/$MAPRED_DIR'/*'
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Add the user-specified CLASSPATH via HADOOP_CLASSPATH
|
|
||||||
# Add it first or last depending on if user has
|
|
||||||
# set env-var HADOOP_USER_CLASSPATH_FIRST
|
|
||||||
if [ "$HADOOP_CLASSPATH" != "" ]; then
|
|
||||||
# Prefix it if its to be preceded
|
|
||||||
if [ "$HADOOP_USER_CLASSPATH_FIRST" != "" ]; then
|
|
||||||
CLASSPATH=${HADOOP_CLASSPATH}:${CLASSPATH}
|
|
||||||
else
|
|
||||||
CLASSPATH=${CLASSPATH}:${HADOOP_CLASSPATH}
|
|
||||||
fi
|
|
||||||
fi
|
fi
|
||||||
|
|
|
@ -15,200 +15,42 @@
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
|
function hadoop_usage
|
||||||
|
{
|
||||||
|
echo "Usage: hadoop-daemon.sh [--config confdir] (start|stop|status) <hadoop-command> <args...>"
|
||||||
|
}
|
||||||
|
|
||||||
# Runs a Hadoop command as a daemon.
|
# let's locate libexec...
|
||||||
#
|
if [[ -n "${HADOOP_PREFIX}" ]]; then
|
||||||
# Environment Variables
|
DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
|
||||||
#
|
else
|
||||||
# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_PREFIX}/conf.
|
this="${BASH_SOURCE-$0}"
|
||||||
# HADOOP_LOG_DIR Where log files are stored. PWD by default.
|
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
|
||||||
# HADOOP_MASTER host:path where hadoop code should be rsync'd from
|
DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
|
||||||
# HADOOP_PID_DIR The pid files are stored. /tmp by default.
|
fi
|
||||||
# HADOOP_IDENT_STRING A string representing this instance of hadoop. $USER by default
|
|
||||||
# HADOOP_NICENESS The scheduling priority for daemons. Defaults to 0.
|
|
||||||
##
|
|
||||||
|
|
||||||
usage="Usage: hadoop-daemon.sh [--config <conf-dir>] [--hosts hostlistfile] [--script script] (start|stop) <hadoop-command> <args...>"
|
HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
|
||||||
|
# shellcheck disable=SC2034
|
||||||
# if no args specified, show usage
|
HADOOP_NEW_CONFIG=true
|
||||||
if [ $# -le 1 ]; then
|
if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then
|
||||||
echo $usage
|
. "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh"
|
||||||
|
else
|
||||||
|
echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
bin=`dirname "${BASH_SOURCE-$0}"`
|
if [[ $# = 0 ]]; then
|
||||||
bin=`cd "$bin"; pwd`
|
hadoop_exit_with_usage 1
|
||||||
|
|
||||||
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
|
|
||||||
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
|
|
||||||
. $HADOOP_LIBEXEC_DIR/hadoop-config.sh
|
|
||||||
|
|
||||||
# get arguments
|
|
||||||
|
|
||||||
#default value
|
|
||||||
hadoopScript="$HADOOP_PREFIX"/bin/hadoop
|
|
||||||
if [ "--script" = "$1" ]
|
|
||||||
then
|
|
||||||
shift
|
|
||||||
hadoopScript=$1
|
|
||||||
shift
|
|
||||||
fi
|
fi
|
||||||
startStop=$1
|
|
||||||
shift
|
daemonmode=$1
|
||||||
command=$1
|
|
||||||
shift
|
shift
|
||||||
|
|
||||||
hadoop_rotate_log ()
|
if [[ -z "${HADOOP_HDFS_HOME}" ]]; then
|
||||||
{
|
hdfsscript="${HADOOP_PREFIX}/bin/hdfs"
|
||||||
log=$1;
|
else
|
||||||
num=5;
|
hdfsscript="${HADOOP_HDFS_HOME}/bin/hdfs"
|
||||||
if [ -n "$2" ]; then
|
|
||||||
num=$2
|
|
||||||
fi
|
|
||||||
if [ -f "$log" ]; then # rotate logs
|
|
||||||
while [ $num -gt 1 ]; do
|
|
||||||
prev=`expr $num - 1`
|
|
||||||
[ -f "$log.$prev" ] && mv "$log.$prev" "$log.$num"
|
|
||||||
num=$prev
|
|
||||||
done
|
|
||||||
mv "$log" "$log.$num";
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
if [ -f "${HADOOP_CONF_DIR}/hadoop-env.sh" ]; then
|
|
||||||
. "${HADOOP_CONF_DIR}/hadoop-env.sh"
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Determine if we're starting a secure datanode, and if so, redefine appropriate variables
|
exec "$hdfsscript" --config "${HADOOP_CONF_DIR}" --daemon "${daemonmode}" "$@"
|
||||||
if [ "$command" == "datanode" ] && [ "$EUID" -eq 0 ] && [ -n "$HADOOP_SECURE_DN_USER" ]; then
|
|
||||||
export HADOOP_PID_DIR=$HADOOP_SECURE_DN_PID_DIR
|
|
||||||
export HADOOP_LOG_DIR=$HADOOP_SECURE_DN_LOG_DIR
|
|
||||||
export HADOOP_IDENT_STRING=$HADOOP_SECURE_DN_USER
|
|
||||||
starting_secure_dn="true"
|
|
||||||
fi
|
|
||||||
|
|
||||||
#Determine if we're starting a privileged NFS, if so, redefine the appropriate variables
|
|
||||||
if [ "$command" == "nfs3" ] && [ "$EUID" -eq 0 ] && [ -n "$HADOOP_PRIVILEGED_NFS_USER" ]; then
|
|
||||||
export HADOOP_PID_DIR=$HADOOP_PRIVILEGED_NFS_PID_DIR
|
|
||||||
export HADOOP_LOG_DIR=$HADOOP_PRIVILEGED_NFS_LOG_DIR
|
|
||||||
export HADOOP_IDENT_STRING=$HADOOP_PRIVILEGED_NFS_USER
|
|
||||||
starting_privileged_nfs="true"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ "$HADOOP_IDENT_STRING" = "" ]; then
|
|
||||||
export HADOOP_IDENT_STRING="$USER"
|
|
||||||
fi
|
|
||||||
|
|
||||||
|
|
||||||
# get log directory
|
|
||||||
if [ "$HADOOP_LOG_DIR" = "" ]; then
|
|
||||||
export HADOOP_LOG_DIR="$HADOOP_PREFIX/logs"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ ! -w "$HADOOP_LOG_DIR" ] ; then
|
|
||||||
mkdir -p "$HADOOP_LOG_DIR"
|
|
||||||
chown $HADOOP_IDENT_STRING $HADOOP_LOG_DIR
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ "$HADOOP_PID_DIR" = "" ]; then
|
|
||||||
HADOOP_PID_DIR=/tmp
|
|
||||||
fi
|
|
||||||
|
|
||||||
# some variables
|
|
||||||
export HADOOP_LOGFILE=hadoop-$HADOOP_IDENT_STRING-$command-$HOSTNAME.log
|
|
||||||
export HADOOP_ROOT_LOGGER=${HADOOP_ROOT_LOGGER:-"INFO,RFA"}
|
|
||||||
export HADOOP_SECURITY_LOGGER=${HADOOP_SECURITY_LOGGER:-"INFO,RFAS"}
|
|
||||||
export HDFS_AUDIT_LOGGER=${HDFS_AUDIT_LOGGER:-"INFO,NullAppender"}
|
|
||||||
log=$HADOOP_LOG_DIR/hadoop-$HADOOP_IDENT_STRING-$command-$HOSTNAME.out
|
|
||||||
pid=$HADOOP_PID_DIR/hadoop-$HADOOP_IDENT_STRING-$command.pid
|
|
||||||
HADOOP_STOP_TIMEOUT=${HADOOP_STOP_TIMEOUT:-5}
|
|
||||||
|
|
||||||
# Set default scheduling priority
|
|
||||||
if [ "$HADOOP_NICENESS" = "" ]; then
|
|
||||||
export HADOOP_NICENESS=0
|
|
||||||
fi
|
|
||||||
|
|
||||||
case $startStop in
|
|
||||||
|
|
||||||
(start)
|
|
||||||
|
|
||||||
[ -w "$HADOOP_PID_DIR" ] || mkdir -p "$HADOOP_PID_DIR"
|
|
||||||
|
|
||||||
if [ -f $pid ]; then
|
|
||||||
if kill -0 `cat $pid` > /dev/null 2>&1; then
|
|
||||||
echo $command running as process `cat $pid`. Stop it first.
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ "$HADOOP_MASTER" != "" ]; then
|
|
||||||
echo rsync from $HADOOP_MASTER
|
|
||||||
rsync -a -e ssh --delete --exclude=.svn --exclude='logs/*' --exclude='contrib/hod/logs/*' $HADOOP_MASTER/ "$HADOOP_PREFIX"
|
|
||||||
fi
|
|
||||||
|
|
||||||
hadoop_rotate_log $log
|
|
||||||
echo starting $command, logging to $log
|
|
||||||
cd "$HADOOP_PREFIX"
|
|
||||||
case $command in
|
|
||||||
namenode|secondarynamenode|datanode|journalnode|dfs|dfsadmin|fsck|balancer|zkfc)
|
|
||||||
if [ -z "$HADOOP_HDFS_HOME" ]; then
|
|
||||||
hdfsScript="$HADOOP_PREFIX"/bin/hdfs
|
|
||||||
else
|
|
||||||
hdfsScript="$HADOOP_HDFS_HOME"/bin/hdfs
|
|
||||||
fi
|
|
||||||
nohup nice -n $HADOOP_NICENESS $hdfsScript --config $HADOOP_CONF_DIR $command "$@" > "$log" 2>&1 < /dev/null &
|
|
||||||
;;
|
|
||||||
(*)
|
|
||||||
nohup nice -n $HADOOP_NICENESS $hadoopScript --config $HADOOP_CONF_DIR $command "$@" > "$log" 2>&1 < /dev/null &
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
echo $! > $pid
|
|
||||||
sleep 1
|
|
||||||
head "$log"
|
|
||||||
# capture the ulimit output
|
|
||||||
if [ "true" = "$starting_secure_dn" ]; then
|
|
||||||
echo "ulimit -a for secure datanode user $HADOOP_SECURE_DN_USER" >> $log
|
|
||||||
# capture the ulimit info for the appropriate user
|
|
||||||
su --shell=/bin/bash $HADOOP_SECURE_DN_USER -c 'ulimit -a' >> $log 2>&1
|
|
||||||
elif [ "true" = "$starting_privileged_nfs" ]; then
|
|
||||||
echo "ulimit -a for privileged nfs user $HADOOP_PRIVILEGED_NFS_USER" >> $log
|
|
||||||
su --shell=/bin/bash $HADOOP_PRIVILEGED_NFS_USER -c 'ulimit -a' >> $log 2>&1
|
|
||||||
else
|
|
||||||
echo "ulimit -a for user $USER" >> $log
|
|
||||||
ulimit -a >> $log 2>&1
|
|
||||||
fi
|
|
||||||
sleep 3;
|
|
||||||
if ! ps -p $! > /dev/null ; then
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
;;
|
|
||||||
|
|
||||||
(stop)
|
|
||||||
|
|
||||||
if [ -f $pid ]; then
|
|
||||||
TARGET_PID=`cat $pid`
|
|
||||||
if kill -0 $TARGET_PID > /dev/null 2>&1; then
|
|
||||||
echo stopping $command
|
|
||||||
kill $TARGET_PID
|
|
||||||
sleep $HADOOP_STOP_TIMEOUT
|
|
||||||
if kill -0 $TARGET_PID > /dev/null 2>&1; then
|
|
||||||
echo "$command did not stop gracefully after $HADOOP_STOP_TIMEOUT seconds: killing with kill -9"
|
|
||||||
kill -9 $TARGET_PID
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
echo no $command to stop
|
|
||||||
fi
|
|
||||||
rm -f $pid
|
|
||||||
else
|
|
||||||
echo no $command to stop
|
|
||||||
fi
|
|
||||||
;;
|
|
||||||
|
|
||||||
(*)
|
|
||||||
echo $usage
|
|
||||||
exit 1
|
|
||||||
;;
|
|
||||||
|
|
||||||
esac
|
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -18,19 +18,34 @@
|
||||||
|
|
||||||
# Run a Hadoop command on all slave hosts.
|
# Run a Hadoop command on all slave hosts.
|
||||||
|
|
||||||
usage="Usage: hadoop-daemons.sh [--config confdir] [--hosts hostlistfile] [start|stop] command args..."
|
function hadoop_usage
|
||||||
|
{
|
||||||
|
echo "Usage: hadoop-daemons.sh [--config confdir] [--hosts hostlistfile] (start|stop|status) <hadoop-command> <args...>"
|
||||||
|
}
|
||||||
|
|
||||||
# if no args specified, show usage
|
this="${BASH_SOURCE-$0}"
|
||||||
if [ $# -le 1 ]; then
|
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
|
||||||
echo $usage
|
|
||||||
|
# let's locate libexec...
|
||||||
|
if [[ -n "${HADOOP_PREFIX}" ]]; then
|
||||||
|
DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
|
||||||
|
else
|
||||||
|
DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
|
||||||
|
fi
|
||||||
|
|
||||||
|
HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
HADOOP_NEW_CONFIG=true
|
||||||
|
if [[ -f "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
|
||||||
|
. "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
|
||||||
|
else
|
||||||
|
echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh." 2>&1
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
bin=`dirname "${BASH_SOURCE-$0}"`
|
if [[ $# = 0 ]]; then
|
||||||
bin=`cd "$bin"; pwd`
|
hadoop_exit_with_usage 1
|
||||||
|
fi
|
||||||
|
|
||||||
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
|
hadoop_connect_to_hosts "${bin}/hadoop-daemon.sh" \
|
||||||
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
|
--config "${HADOOP_CONF_DIR}" "$@"
|
||||||
. $HADOOP_LIBEXEC_DIR/hadoop-config.sh
|
|
||||||
|
|
||||||
exec "$bin/slaves.sh" --config $HADOOP_CONF_DIR cd "$HADOOP_PREFIX" \; "$bin/hadoop-daemon.sh" --config $HADOOP_CONF_DIR "$@"
|
|
||||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -0,0 +1,93 @@
|
||||||
|
# Copyright 2014 The Apache Software Foundation
|
||||||
|
#
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
##
|
||||||
|
## VENDORS!
|
||||||
|
##
|
||||||
|
## This is where you can redefine the layout of Hadoop directories
|
||||||
|
## and expect to be reasonably compatible. Needless to say, this
|
||||||
|
## is expert level stuff and one needs to tread carefully.
|
||||||
|
##
|
||||||
|
## If you move HADOOP_LIBEXEC_DIR from some location that
|
||||||
|
## isn't bin/../libexec, you MUST define either HADOOP_LIBEXEC_DIR
|
||||||
|
## or have HADOOP_PREFIX/libexec/hadoop-config.sh and
|
||||||
|
## HADOOP_PREFIX/libexec/hadoop-layout.sh (this file) exist.
|
||||||
|
|
||||||
|
## NOTE:
|
||||||
|
##
|
||||||
|
## hadoop-functions.sh gets executed BEFORE this file. So you can
|
||||||
|
## redefine all of those functions here.
|
||||||
|
##
|
||||||
|
## *-env.sh get executed AFTER this file but generally too late to
|
||||||
|
## override the settings (but not the functions!) here. However, this
|
||||||
|
## also means you cannot use things like HADOOP_CONF_DIR for these
|
||||||
|
## definitions.
|
||||||
|
|
||||||
|
####
|
||||||
|
# Common disk layout
|
||||||
|
####
|
||||||
|
|
||||||
|
# Default location for the common/core Hadoop project
|
||||||
|
# export HADOOP_COMMON_HOME=$HADOOP_PREFIX
|
||||||
|
|
||||||
|
# Relative locations where components under HADOOP_COMMON_HOME are located
|
||||||
|
# export HADOOP_COMMON_DIR="share/hadoop/common"
|
||||||
|
# export HADOOP_COMMON_LIB_JARS_DIR="share/hadoop/common/lib"
|
||||||
|
# export HADOOP_COMMON_LIB_NATIVE_DIR="lib/native"
|
||||||
|
|
||||||
|
####
|
||||||
|
# HDFS disk layout
|
||||||
|
####
|
||||||
|
|
||||||
|
# Default location for the HDFS subproject
|
||||||
|
# export HADOOP_HDFS_HOME=$HADOOP_PREFIX
|
||||||
|
|
||||||
|
# Relative locations where components under HADOOP_HDFS_HOME are located
|
||||||
|
# export HDFS_DIR="share/hadoop/hdfs"
|
||||||
|
# export HDFS_LIB_JARS_DIR="share/hadoop/hdfs/lib"
|
||||||
|
|
||||||
|
####
|
||||||
|
# YARN disk layout
|
||||||
|
####
|
||||||
|
|
||||||
|
# Default location for the YARN subproject
|
||||||
|
# export HADOOP_YARN_HOME=$HADOOP_PREFIX
|
||||||
|
|
||||||
|
# Relative locations where components under HADOOP_YARN_HOME are located
|
||||||
|
# export YARN_DIR="share/hadoop/yarn"
|
||||||
|
# export YARN_LIB_JARS_DIR="share/hadoop/yarn/lib"
|
||||||
|
|
||||||
|
# Default location for the MapReduce subproject
|
||||||
|
# export HADOOP_MAPRED_HOME=$HADOOP_PREFIX
|
||||||
|
|
||||||
|
####
|
||||||
|
# MapReduce disk layout
|
||||||
|
####
|
||||||
|
|
||||||
|
# Relative locations where components under HADOOP_MAPRED_HOME are located
|
||||||
|
# export MAPRED_DIR="share/hadoop/mapreduce"
|
||||||
|
# export MAPRED_LIB_JARS_DIR="share/hadoop/mapreduce/lib"
|
||||||
|
|
||||||
|
####
|
||||||
|
# Misc paths
|
||||||
|
####
|
||||||
|
|
||||||
|
# setup a default TOOL_PATH, where things like distcp lives
|
||||||
|
# note that this path only gets added for certain commands and not
|
||||||
|
# part of the general classpath
|
||||||
|
# export TOOL_PATH="$HADOOP_PREFIX/share/hadoop/tools/lib/*"
|
|
@ -15,47 +15,28 @@
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
|
# This script runs the hadoop core commands.
|
||||||
# The Hadoop record compiler
|
this="${BASH_SOURCE-$0}"
|
||||||
#
|
bin=$(cd -P -- "$(dirname -- "$this")" >/dev/null && pwd -P)
|
||||||
# Environment Variables
|
script="$(basename -- "$this")"
|
||||||
#
|
this="$bin/$script"
|
||||||
# JAVA_HOME The java implementation to use. Overrides JAVA_HOME.
|
|
||||||
#
|
|
||||||
# HADOOP_OPTS Extra Java runtime options.
|
|
||||||
#
|
|
||||||
# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_PREFIX}/conf.
|
|
||||||
#
|
|
||||||
|
|
||||||
bin=`dirname "${BASH_SOURCE-$0}"`
|
|
||||||
bin=`cd "$bin"; pwd`
|
|
||||||
|
|
||||||
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
|
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
|
||||||
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
|
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
|
||||||
. $HADOOP_LIBEXEC_DIR/hadoop-config.sh
|
HADOOP_NEW_CONFIG=true
|
||||||
|
. "$HADOOP_LIBEXEC_DIR/hadoop-config.sh"
|
||||||
|
|
||||||
if [ -f "${HADOOP_CONF_DIR}/hadoop-env.sh" ]; then
|
if [ $# = 0 ]; then
|
||||||
. "${HADOOP_CONF_DIR}/hadoop-env.sh"
|
hadoop_exit_with_usage 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# some Java parameters
|
|
||||||
if [ "$JAVA_HOME" != "" ]; then
|
|
||||||
#echo "run java in $JAVA_HOME"
|
|
||||||
JAVA_HOME=$JAVA_HOME
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ "$JAVA_HOME" = "" ]; then
|
|
||||||
echo "Error: JAVA_HOME is not set."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
JAVA=$JAVA_HOME/bin/java
|
|
||||||
JAVA_HEAP_MAX=-Xmx1000m
|
|
||||||
|
|
||||||
# restore ordinary behaviour
|
|
||||||
unset IFS
|
|
||||||
|
|
||||||
CLASS='org.apache.hadoop.record.compiler.generated.Rcc'
|
CLASS='org.apache.hadoop.record.compiler.generated.Rcc'
|
||||||
|
|
||||||
# run it
|
# Always respect HADOOP_OPTS and HADOOP_CLIENT_OPTS
|
||||||
exec "$JAVA" $HADOOP_OPTS -classpath "$CLASSPATH" $CLASS "$@"
|
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
|
||||||
|
|
||||||
|
hadoop_add_param HADOOP_OPTS Xmx "$JAVA_HEAP_MAX"
|
||||||
|
|
||||||
|
hadoop_finalize
|
||||||
|
export CLASSPATH
|
||||||
|
hadoop_java_exec rcc "${CLASS}" "$@"
|
||||||
|
|
|
@ -27,38 +27,33 @@
|
||||||
# HADOOP_SSH_OPTS Options passed to ssh when running remote commands.
|
# HADOOP_SSH_OPTS Options passed to ssh when running remote commands.
|
||||||
##
|
##
|
||||||
|
|
||||||
usage="Usage: slaves.sh [--config confdir] command..."
|
function hadoop_usage {
|
||||||
|
echo "Usage: slaves.sh [--config confdir] command..."
|
||||||
|
}
|
||||||
|
|
||||||
# if no args specified, show usage
|
# let's locate libexec...
|
||||||
if [ $# -le 0 ]; then
|
if [[ -n "${HADOOP_PREFIX}" ]]; then
|
||||||
echo $usage
|
DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
|
||||||
|
else
|
||||||
|
this="${BASH_SOURCE-$0}"
|
||||||
|
bin=$(cd -P -- "$(dirname -- "${this}")" >dev/null && pwd -P)
|
||||||
|
DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
|
||||||
|
fi
|
||||||
|
|
||||||
|
HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
HADOOP_NEW_CONFIG=true
|
||||||
|
if [[ -f "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
|
||||||
|
. "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
|
||||||
|
else
|
||||||
|
echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh." 2>&1
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
bin=`dirname "${BASH_SOURCE-$0}"`
|
# if no args specified, show usage
|
||||||
bin=`cd "$bin"; pwd`
|
if [[ $# -le 0 ]]; then
|
||||||
|
hadoop_exit_with_usage 1
|
||||||
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
|
|
||||||
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
|
|
||||||
. $HADOOP_LIBEXEC_DIR/hadoop-config.sh
|
|
||||||
|
|
||||||
|
|
||||||
# Where to start the script, see hadoop-config.sh
|
|
||||||
# (it set up the variables based on command line options)
|
|
||||||
if [ "$HADOOP_SLAVE_NAMES" != '' ] ; then
|
|
||||||
SLAVE_NAMES=$HADOOP_SLAVE_NAMES
|
|
||||||
else
|
|
||||||
SLAVE_FILE=${HADOOP_SLAVES:-${HADOOP_CONF_DIR}/slaves}
|
|
||||||
SLAVE_NAMES=$(cat "$SLAVE_FILE" | sed 's/#.*$//;/^$/d')
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# start the daemons
|
hadoop_connect_to_hosts "$@"
|
||||||
for slave in $SLAVE_NAMES ; do
|
|
||||||
ssh $HADOOP_SSH_OPTS $slave $"${@// /\\ }" \
|
|
||||||
2>&1 | sed "s/^/$slave: /" &
|
|
||||||
if [ "$HADOOP_SLAVE_SLEEP" != "" ]; then
|
|
||||||
sleep $HADOOP_SLAVE_SLEEP
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
wait
|
|
||||||
|
|
|
@ -15,24 +15,38 @@
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
|
echo "This script is deprecated. Use start-dfs.sh and start-yarn.sh instead."
|
||||||
|
exit 1
|
||||||
|
|
||||||
# Start all hadoop daemons. Run this on master node.
|
|
||||||
|
|
||||||
echo "This script is Deprecated. Instead use start-dfs.sh and start-yarn.sh"
|
|
||||||
|
|
||||||
bin=`dirname "${BASH_SOURCE-$0}"`
|
# let's locate libexec...
|
||||||
bin=`cd "$bin"; pwd`
|
if [[ -n "${HADOOP_PREFIX}" ]]; then
|
||||||
|
DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
|
||||||
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
|
else
|
||||||
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
|
this="${BASH_SOURCE-$0}"
|
||||||
. $HADOOP_LIBEXEC_DIR/hadoop-config.sh
|
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
|
||||||
|
DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
|
||||||
|
fi
|
||||||
|
|
||||||
|
HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
HADOOP_NEW_CONFIG=true
|
||||||
|
if [[ -f "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
|
||||||
|
. "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
|
||||||
|
else
|
||||||
|
echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh." 2>&1
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
# start hdfs daemons if hdfs is present
|
# start hdfs daemons if hdfs is present
|
||||||
if [ -f "${HADOOP_HDFS_HOME}"/sbin/start-dfs.sh ]; then
|
if [[ -f "${HADOOP_HDFS_HOME}/sbin/start-dfs.sh" ]]; then
|
||||||
"${HADOOP_HDFS_HOME}"/sbin/start-dfs.sh --config $HADOOP_CONF_DIR
|
"${HADOOP_HDFS_HOME}/sbin/start-dfs.sh" --config "${HADOOP_CONF_DIR}"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# start yarn daemons if yarn is present
|
# start yarn daemons if yarn is present
|
||||||
if [ -f "${HADOOP_YARN_HOME}"/sbin/start-yarn.sh ]; then
|
if [[ -f "${HADOOP_YARN_HOME}/sbin/start-yarn.sh" ]]; then
|
||||||
"${HADOOP_YARN_HOME}"/sbin/start-yarn.sh --config $HADOOP_CONF_DIR
|
"${HADOOP_YARN_HOME}/sbin/start-yarn.sh" --config "${HADOOP_CONF_DIR}"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -18,21 +18,35 @@
|
||||||
|
|
||||||
# Stop all hadoop daemons. Run this on master node.
|
# Stop all hadoop daemons. Run this on master node.
|
||||||
|
|
||||||
echo "This script is Deprecated. Instead use stop-dfs.sh and stop-yarn.sh"
|
echo "This script is deprecated. Use stop-dfs.sh and stop-yarn.sh instead."
|
||||||
|
exit 1
|
||||||
|
|
||||||
bin=`dirname "${BASH_SOURCE-$0}"`
|
# let's locate libexec...
|
||||||
bin=`cd "$bin"; pwd`
|
if [[ -n "${HADOOP_PREFIX}" ]]; then
|
||||||
|
DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
|
||||||
|
else
|
||||||
|
this="${BASH_SOURCE-$0}"
|
||||||
|
bin=$(cd -P -- "$(dirname -- "${this}")" >dev/null && pwd -P)
|
||||||
|
DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
|
||||||
|
fi
|
||||||
|
|
||||||
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
|
HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
|
||||||
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
|
# shellcheck disable=SC2034
|
||||||
. $HADOOP_LIBEXEC_DIR/hadoop-config.sh
|
HADOOP_NEW_CONFIG=true
|
||||||
|
if [[ -f "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
|
||||||
|
. "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
|
||||||
|
else
|
||||||
|
echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh." 2>&1
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
# stop hdfs daemons if hdfs is present
|
# stop hdfs daemons if hdfs is present
|
||||||
if [ -f "${HADOOP_HDFS_HOME}"/sbin/stop-dfs.sh ]; then
|
if [[ -f "${HADOOP_HDFS_HOME}/sbin/stop-dfs.sh" ]]; then
|
||||||
"${HADOOP_HDFS_HOME}"/sbin/stop-dfs.sh --config $HADOOP_CONF_DIR
|
"${HADOOP_HDFS_HOME}/sbin/stop-dfs.sh" --config "${HADOOP_CONF_DIR}"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# stop yarn daemons if yarn is present
|
# stop yarn daemons if yarn is present
|
||||||
if [ -f "${HADOOP_HDFS_HOME}"/sbin/stop-yarn.sh ]; then
|
if [[ -f "${HADOOP_HDFS_HOME}/sbin/stop-yarn.sh" ]]; then
|
||||||
"${HADOOP_HDFS_HOME}"/sbin/stop-yarn.sh --config $HADOOP_CONF_DIR
|
"${HADOOP_HDFS_HOME}/sbin/stop-yarn.sh" --config "${HADOOP_CONF_DIR}"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
#
|
||||||
# Licensed to the Apache Software Foundation (ASF) under one
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
# or more contributor license agreements. See the NOTICE file
|
# or more contributor license agreements. See the NOTICE file
|
||||||
# distributed with this work for additional information
|
# distributed with this work for additional information
|
||||||
|
@ -16,71 +17,393 @@
|
||||||
|
|
||||||
# Set Hadoop-specific environment variables here.
|
# Set Hadoop-specific environment variables here.
|
||||||
|
|
||||||
# The only required environment variable is JAVA_HOME. All others are
|
##
|
||||||
# optional. When running a distributed configuration it is best to
|
## THIS FILE ACTS AS THE MASTER FILE FOR ALL HADOOP PROJECTS.
|
||||||
# set JAVA_HOME in this file, so that it is correctly defined on
|
## SETTINGS HERE WILL BE READ BY ALL HADOOP COMMANDS. THEREFORE,
|
||||||
# remote nodes.
|
## ONE CAN USE THIS FILE TO SET YARN, HDFS, AND MAPREDUCE
|
||||||
|
## CONFIGURATION OPTIONS INSTEAD OF xxx-env.sh.
|
||||||
|
##
|
||||||
|
## Precedence rules:
|
||||||
|
##
|
||||||
|
## {yarn-env.sh|hdfs-env.sh} > hadoop-env.sh > hard-coded defaults
|
||||||
|
##
|
||||||
|
## {YARN_xyz|HDFS_xyz} > HADOOP_xyz > hard-coded defaults
|
||||||
|
##
|
||||||
|
|
||||||
|
# Many of the options here are built from the perspective that users
|
||||||
|
# may want to provide OVERWRITING values on the command line.
|
||||||
|
# For example:
|
||||||
|
#
|
||||||
|
# JAVA_HOME=/usr/java/testing hdfs dfs -ls
|
||||||
|
#
|
||||||
|
# Therefore, the vast majority (BUT NOT ALL!) of these defaults
|
||||||
|
# are configured for substitution and not append. If you would
|
||||||
|
# like append, you'll # need to modify this file accordingly.
|
||||||
|
|
||||||
|
###
|
||||||
|
# Generic settings for HADOOP
|
||||||
|
###
|
||||||
|
|
||||||
|
# Technically, the only required environment variable is JAVA_HOME.
|
||||||
|
# All others are optional. However, our defaults are probably not
|
||||||
|
# your defaults. Many sites configure these options outside of Hadoop,
|
||||||
|
# such as in /etc/profile.d
|
||||||
|
|
||||||
# The java implementation to use.
|
# The java implementation to use.
|
||||||
export JAVA_HOME=${JAVA_HOME}
|
export JAVA_HOME=${JAVA_HOME:-"hadoop-env.sh is not configured"}
|
||||||
|
|
||||||
# The jsvc implementation to use. Jsvc is required to run secure datanodes.
|
# Location of Hadoop's configuration information. i.e., where this
|
||||||
#export JSVC_HOME=${JSVC_HOME}
|
# file is probably living. You will almost certainly want to set
|
||||||
|
# this in /etc/profile.d or equivalent.
|
||||||
|
# export HADOOP_CONF_DIR=$HADOOP_PREFIX/etc/hadoop
|
||||||
|
|
||||||
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/etc/hadoop"}
|
# The maximum amount of heap to use, in MB. Default is 1024.
|
||||||
|
# export HADOOP_HEAPSIZE=1024
|
||||||
|
|
||||||
# Extra Java CLASSPATH elements. Automatically insert capacity-scheduler.
|
# Extra Java runtime options for all Hadoop commands. We don't support
|
||||||
for f in $HADOOP_HOME/contrib/capacity-scheduler/*.jar; do
|
# IPv6 yet/still, so by default we set preference to IPv4.
|
||||||
if [ "$HADOOP_CLASSPATH" ]; then
|
# export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true"
|
||||||
export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$f
|
|
||||||
else
|
|
||||||
export HADOOP_CLASSPATH=$f
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
# The maximum amount of heap to use, in MB. Default is 1000.
|
# Some parts of the shell code may do special things dependent upon
|
||||||
#export HADOOP_HEAPSIZE=
|
# the operating system. We have to set this here. See the next
|
||||||
#export HADOOP_NAMENODE_INIT_HEAPSIZE=""
|
# section as to why....
|
||||||
|
export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)}
|
||||||
|
|
||||||
# Extra Java runtime options. Empty by default.
|
|
||||||
export HADOOP_OPTS="$HADOOP_OPTS -Djava.net.preferIPv4Stack=true"
|
|
||||||
|
|
||||||
MAC_OSX=false
|
# Under certain conditions, Java on OS X will throw SCDynamicStore errors
|
||||||
case "`uname`" in
|
# in the system logs.
|
||||||
Darwin*) MAC_OSX=true;;
|
# See HADOOP-8719 for more information. If you need Kerberos
|
||||||
|
# support on OS X, you'll want to change/remove this extra bit.
|
||||||
|
case ${HADOOP_OS_TYPE} in
|
||||||
|
Darwin*)
|
||||||
|
export HADOOP_OPTS="${HADOOP_OPTS} -Djava.security.krb5.realm= "
|
||||||
|
export HADOOP_OPTS="${HADOOP_OPTS} -Djava.security.krb5.kdc= "
|
||||||
|
export HADOOP_OPTS="${HADOOP_OPTS} -Djava.security.krb5.conf= "
|
||||||
|
;;
|
||||||
esac
|
esac
|
||||||
if $MAC_OSX; then
|
|
||||||
export HADOOP_OPTS="$HADOOP_OPTS -Djava.security.krb5.realm= -Djava.security.krb5.kdc="
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Command specific options appended to HADOOP_OPTS when specified
|
# Extra Java runtime options for Hadoop clients (i.e., hdfs dfs -blah)
|
||||||
export HADOOP_NAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_NAMENODE_OPTS"
|
# These get added to HADOOP_OPTS for such commands. In most cases,
|
||||||
export HADOOP_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS $HADOOP_DATANODE_OPTS"
|
# this should be left empty and let users supply it on the
|
||||||
|
# command line.
|
||||||
|
# extra HADOOP_CLIENT_OPTS=""
|
||||||
|
|
||||||
export HADOOP_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_SECONDARYNAMENODE_OPTS"
|
#
|
||||||
|
# A note about classpaths.
|
||||||
|
#
|
||||||
|
# The classpath is configured such that entries are stripped prior
|
||||||
|
# to handing to Java based either upon duplication or non-existence.
|
||||||
|
# Wildcards and/or directories are *NOT* expanded as the
|
||||||
|
# de-duplication is fairly simple. So if two directories are in
|
||||||
|
# the classpath that both contain awesome-methods-1.0.jar,
|
||||||
|
# awesome-methods-1.0.jar will still be seen by java. But if
|
||||||
|
# the classpath specifically has awesome-methods-1.0.jar from the
|
||||||
|
# same directory listed twice, the last one will be removed.
|
||||||
|
#
|
||||||
|
|
||||||
export HADOOP_NFS3_OPTS="$HADOOP_NFS3_OPTS"
|
# An additional, custom CLASSPATH. This is really meant for
|
||||||
export HADOOP_PORTMAP_OPTS="-Xmx512m $HADOOP_PORTMAP_OPTS"
|
# end users, but as an administrator, one might want to push
|
||||||
|
# something extra in here too, such as the jar to the topology
|
||||||
|
# method. Just be sure to append to the existing HADOOP_USER_CLASSPATH
|
||||||
|
# so end users have a way to add stuff.
|
||||||
|
# export HADOOP_USER_CLASSPATH="/some/cool/path/on/your/machine"
|
||||||
|
|
||||||
# The following applies to multiple commands (fs, dfs, fsck, distcp etc)
|
# Should HADOOP_USER_CLASSPATH be first in the official CLASSPATH?
|
||||||
export HADOOP_CLIENT_OPTS="-Xmx512m $HADOOP_CLIENT_OPTS"
|
# export HADOOP_USER_CLASSPATH_FIRST="yes"
|
||||||
#HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData $HADOOP_JAVA_PLATFORM_OPTS"
|
|
||||||
|
|
||||||
# On secure datanodes, user to run the datanode as after dropping privileges
|
###
|
||||||
export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER}
|
# Options for remote shell connectivity
|
||||||
|
###
|
||||||
|
|
||||||
# Where log files are stored. $HADOOP_HOME/logs by default.
|
# There are some optional components of hadoop that allow for
|
||||||
#export HADOOP_LOG_DIR=${HADOOP_LOG_DIR}/$USER
|
# command and control of remote hosts. For example,
|
||||||
|
# start-dfs.sh will attempt to bring up all NNs, DNS, etc.
|
||||||
|
|
||||||
# Where log files are stored in the secure data environment.
|
# Options to pass to SSH when one of the "log into a host and
|
||||||
export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}
|
# start/stop daemons" scripts is executed
|
||||||
|
# export HADOOP_SSH_OPTS="-o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=10s"
|
||||||
|
|
||||||
# The directory where pid files are stored. /tmp by default.
|
# The built-in ssh handler will limit itself to 10 simultaneous connections.
|
||||||
# NOTE: this should be set to a directory that can only be written to by
|
# For pdsh users, this sets the fanout size ( -f )
|
||||||
# the user that will run the hadoop daemons. Otherwise there is the
|
# Change this to increase/decrease as necessary.
|
||||||
# potential for a symlink attack.
|
# export HADOOP_SSH_PARALLEL=10
|
||||||
export HADOOP_PID_DIR=${HADOOP_PID_DIR}
|
|
||||||
export HADOOP_SECURE_DN_PID_DIR=${HADOOP_PID_DIR}
|
# Filename which contains all of the hosts for any remote execution
|
||||||
|
# helper scripts # such as slaves.sh, start-dfs.sh, etc.
|
||||||
|
# export HADOOP_SLAVES="${HADOOP_CONF_DIR}/slaves"
|
||||||
|
|
||||||
|
###
|
||||||
|
# Options for all daemons
|
||||||
|
###
|
||||||
|
#
|
||||||
|
|
||||||
|
#
|
||||||
|
# You can define variables right here and then re-use them later on.
|
||||||
|
# For example, it is common to use the same garbage collection settings
|
||||||
|
# for all the daemons. So we could define:
|
||||||
|
#
|
||||||
|
# export HADOOP_GC_SETTINGS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps"
|
||||||
|
#
|
||||||
|
# .. and then use it as per the b option under the namenode.
|
||||||
|
|
||||||
|
# Where (primarily) daemon log files are stored.
|
||||||
|
# $HADOOP_PREFIX/logs by default.
|
||||||
|
# export HADOOP_LOG_DIR=${HADOOP_PREFIX}/logs
|
||||||
|
|
||||||
# A string representing this instance of hadoop. $USER by default.
|
# A string representing this instance of hadoop. $USER by default.
|
||||||
export HADOOP_IDENT_STRING=$USER
|
# This is used in writing log and pid files, so keep that in mind!
|
||||||
|
# export HADOOP_IDENT_STRING=$USER
|
||||||
|
|
||||||
|
# How many seconds to pause after stopping a daemon
|
||||||
|
# export HADOOP_STOP_TIMEOUT=5
|
||||||
|
|
||||||
|
# Where pid files are stored. /tmp by default.
|
||||||
|
# export HADOOP_PID_DIR=/tmp
|
||||||
|
|
||||||
|
# Default log level and output location
|
||||||
|
# This sets the hadoop.root.logger property
|
||||||
|
# export HADOOP_ROOT_LOGGER=INFO,console
|
||||||
|
|
||||||
|
# Default log level for daemons spawned explicitly by hadoop-daemon.sh
|
||||||
|
# This sets the hadoop.root.logger property
|
||||||
|
# export HADOOP_DAEMON_ROOT_LOGGER=INFO,RFA
|
||||||
|
|
||||||
|
# Default log level and output location for security-related messages.
|
||||||
|
# It sets -Dhadoop.security.logger on the command line.
|
||||||
|
# You will almost certainly want to change this on a per-daemon basis!
|
||||||
|
# export HADOOP_SECURITY_LOGGER=INFO,NullAppender
|
||||||
|
|
||||||
|
# Default log level for file system audit messages.
|
||||||
|
# It sets -Dhdfs.audit.logger on the command line.
|
||||||
|
# You will almost certainly want to change this on a per-daemon basis!
|
||||||
|
# export HADOOP_AUDIT_LOGGER=INFO,NullAppender
|
||||||
|
|
||||||
|
# Default process priority level
|
||||||
|
# Note that sub-processes will also run at this level!
|
||||||
|
# export HADOOP_NICENESS=0
|
||||||
|
|
||||||
|
# Default name for the service level authorization file
|
||||||
|
# export HADOOP_POLICYFILE="hadoop-policy.xml"
|
||||||
|
|
||||||
|
###
|
||||||
|
# Secure/privileged execution
|
||||||
|
###
|
||||||
|
|
||||||
|
#
|
||||||
|
# Out of the box, Hadoop uses jsvc from Apache Commons to launch daemons
|
||||||
|
# on privileged ports. This functionality can be replaced by providing
|
||||||
|
# custom functions. See hadoop-functions.sh for more information.
|
||||||
|
#
|
||||||
|
|
||||||
|
# The jsvc implementation to use. Jsvc is required to run secure datanodes.
|
||||||
|
# export JSVC_HOME=/usr/bin
|
||||||
|
|
||||||
|
#
|
||||||
|
# This directory contains pids for secure and privileged processes.
|
||||||
|
#export HADOOP_SECURE_PID_DIR=${HADOOP_PID_DIR}
|
||||||
|
|
||||||
|
#
|
||||||
|
# This directory contains the logs for secure and privileged processes.
|
||||||
|
# export HADOOP_SECURE_LOG=${HADOOP_LOG_DIR}
|
||||||
|
|
||||||
|
#
|
||||||
|
# When running a secure daemon, the default value of HADOOP_IDENT_STRING
|
||||||
|
# ends up being a bit bogus. Therefore, by default, the code will
|
||||||
|
# replace HADOOP_IDENT_STRING with HADOOP_SECURE_xx_USER. If you want
|
||||||
|
# to keep HADOOP_IDENT_STRING untouched, then uncomment this line.
|
||||||
|
# export HADOOP_SECURE_IDENT_PRESERVE="true"
|
||||||
|
|
||||||
|
###
|
||||||
|
# NameNode specific parameters
|
||||||
|
###
|
||||||
|
# Specify the JVM options to be used when starting the NameNode.
|
||||||
|
# These options will be appended to the options specified as HADOOP_OPTS
|
||||||
|
# and therefore may override any similar flags set in HADOOP_OPTS
|
||||||
|
#
|
||||||
|
# a) Set JMX options
|
||||||
|
# export HADOOP_NAMENODE_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=1026"
|
||||||
|
#
|
||||||
|
# b) Set garbage collection logs
|
||||||
|
# export HADOOP_NAMENODE_OPTS="${HADOOP_GC_SETTINGS} -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')"
|
||||||
|
#
|
||||||
|
# c) ... or set them directly
|
||||||
|
# export HADOOP_NAMENODE_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')"
|
||||||
|
|
||||||
|
# this is the default:
|
||||||
|
# export HADOOP_NAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS -Dhdfs.audit.logger=INFO,NullAppender"
|
||||||
|
|
||||||
|
###
|
||||||
|
# SecondaryNameNode specific parameters
|
||||||
|
###
|
||||||
|
# Specify the JVM options to be used when starting the SecondaryNameNode.
|
||||||
|
# These options will be appended to the options specified as HADOOP_OPTS
|
||||||
|
# and therefore may override any similar flags set in HADOOP_OPTS
|
||||||
|
#
|
||||||
|
# This is the default:
|
||||||
|
# export HADOOP_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS -Dhdfs.audit.logger=INFO,NullAppender"
|
||||||
|
|
||||||
|
###
|
||||||
|
# DataNode specific parameters
|
||||||
|
###
|
||||||
|
# Specify the JVM options to be used when starting the DataNode.
|
||||||
|
# These options will be appended to the options specified as HADOOP_OPTS
|
||||||
|
# and therefore may override any similar flags set in HADOOP_OPTS
|
||||||
|
#
|
||||||
|
# This is the default:
|
||||||
|
# export HADOOP_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS"
|
||||||
|
|
||||||
|
# On secure datanodes, user to run the datanode as after dropping privileges
|
||||||
|
# This **MUST** be uncommented to enable secure HDFS!
|
||||||
|
# export HADOOP_SECURE_DN_USER=hdfs
|
||||||
|
|
||||||
|
# Supplemental options for secure datanodes
|
||||||
|
# By default, we use jsvc which needs to know to launch a
|
||||||
|
# server jvm.
|
||||||
|
# export HADOOP_DN_SECURE_EXTRA_OPTS="-jvm server"
|
||||||
|
|
||||||
|
# Where datanode log files are stored in the secure data environment.
|
||||||
|
# export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_SECURE_LOG_DIR}
|
||||||
|
|
||||||
|
# Where datanode pid files are stored in the secure data environment.
|
||||||
|
# export HADOOP_SECURE_DN_PID_DIR=${HADOOP_SECURE_PID_DIR}
|
||||||
|
|
||||||
|
###
|
||||||
|
# NFS3 Gateway specific parameters
|
||||||
|
###
|
||||||
|
# Specify the JVM options to be used when starting the NFS3 Gateway.
|
||||||
|
# These options will be appended to the options specified as HADOOP_OPTS
|
||||||
|
# and therefore may override any similar flags set in HADOOP_OPTS
|
||||||
|
#
|
||||||
|
# export HADOOP_NFS3_OPTS=""
|
||||||
|
|
||||||
|
# Specify the JVM options to be used when starting the Hadoop portmapper.
|
||||||
|
# These options will be appended to the options specified as HADOOP_OPTS
|
||||||
|
# and therefore may override any similar flags set in HADOOP_OPTS
|
||||||
|
#
|
||||||
|
# export HADOOP_PORTMAP_OPTS="-Xmx512m"
|
||||||
|
|
||||||
|
# Supplemental options for priviliged gateways
|
||||||
|
# By default, we use jsvc which needs to know to launch a
|
||||||
|
# server jvm.
|
||||||
|
# export HADOOP_NFS3_SECURE_EXTRA_OPTS="-jvm server"
|
||||||
|
|
||||||
|
# On privileged gateways, user to run the gateway as after dropping privileges
|
||||||
|
# export HADOOP_PRIVILEGED_NFS_USER=nfsserver
|
||||||
|
|
||||||
|
###
|
||||||
|
# ZKFailoverController specific parameters
|
||||||
|
###
|
||||||
|
# Specify the JVM options to be used when starting the ZKFailoverController.
|
||||||
|
# These options will be appended to the options specified as HADOOP_OPTS
|
||||||
|
# and therefore may override any similar flags set in HADOOP_OPTS
|
||||||
|
#
|
||||||
|
# export HADOOP_ZKFC_OPTS=""
|
||||||
|
|
||||||
|
###
|
||||||
|
# QuorumJournalNode specific parameters
|
||||||
|
###
|
||||||
|
# Specify the JVM options to be used when starting the QuorumJournalNode.
|
||||||
|
# These options will be appended to the options specified as HADOOP_OPTS
|
||||||
|
# and therefore may override any similar flags set in HADOOP_OPTS
|
||||||
|
#
|
||||||
|
# export HADOOP_JOURNALNODE_OPTS=""
|
||||||
|
|
||||||
|
###
|
||||||
|
# HDFS Balancer specific parameters
|
||||||
|
###
|
||||||
|
# Specify the JVM options to be used when starting the HDFS Balancer.
|
||||||
|
# These options will be appended to the options specified as HADOOP_OPTS
|
||||||
|
# and therefore may override any similar flags set in HADOOP_OPTS
|
||||||
|
#
|
||||||
|
# export HADOOP_BALANCER_OPTS=""
|
||||||
|
|
||||||
|
###
|
||||||
|
# Advanced Users Only!
|
||||||
|
###
|
||||||
|
|
||||||
|
#
|
||||||
|
# When building Hadoop, you can add the class paths to your commands
|
||||||
|
# via this special env var:
|
||||||
|
# HADOOP_ENABLE_BUILD_PATHS="true"
|
||||||
|
|
||||||
|
# You can do things like replace parts of the shell underbelly.
|
||||||
|
# Most of this code is in hadoop-functions.sh.
|
||||||
|
#
|
||||||
|
#
|
||||||
|
# For example, if you want to add compression to the rotation
|
||||||
|
# menthod for the .out files that daemons generate, you can do
|
||||||
|
# that by redefining the hadoop_rotate_log function by
|
||||||
|
# uncommenting this code block:
|
||||||
|
|
||||||
|
#function hadoop_rotate_log
|
||||||
|
#{
|
||||||
|
# #
|
||||||
|
# # log rotation (mainly used for .out files)
|
||||||
|
# # Users are likely to replace this one for something
|
||||||
|
# # that gzips or uses dates or who knows what.
|
||||||
|
# #
|
||||||
|
# # be aware that &1 and &2 might go through here
|
||||||
|
# # so don't do anything too crazy...
|
||||||
|
# #
|
||||||
|
# local log=$1;
|
||||||
|
# local num=${2:-5};
|
||||||
|
#
|
||||||
|
# if [[ -f "${log}" ]]; then # rotate logs
|
||||||
|
# while [[ ${num} -gt 1 ]]; do
|
||||||
|
# #shellcheck disable=SC2086
|
||||||
|
# let prev=${num}-1
|
||||||
|
# if [[ -f "${log}.${prev}" ]]; then
|
||||||
|
# mv "${log}.${prev}" "${log}.${num}"
|
||||||
|
# fi
|
||||||
|
# num=${prev}
|
||||||
|
# done
|
||||||
|
# mv "${log}" "${log}.${num}"
|
||||||
|
# gzip -9 "${log}.${num}"
|
||||||
|
# fi
|
||||||
|
#}
|
||||||
|
#
|
||||||
|
#
|
||||||
|
# Another example: finding java
|
||||||
|
#
|
||||||
|
# By default, Hadoop assumes that $JAVA_HOME is always defined
|
||||||
|
# outside of its configuration. Eons ago, Apple standardized
|
||||||
|
# on a helper program called java_home to find it for you.
|
||||||
|
#
|
||||||
|
#function hadoop_java_setup
|
||||||
|
#{
|
||||||
|
#
|
||||||
|
# if [[ -z "${JAVA_HOME}" ]]; then
|
||||||
|
# case $HADOOP_OS_TYPE in
|
||||||
|
# Darwin*)
|
||||||
|
# JAVA_HOME=$(/usr/libexec/java_home)
|
||||||
|
# ;;
|
||||||
|
# esac
|
||||||
|
# fi
|
||||||
|
#
|
||||||
|
# # Bail if we did not detect it
|
||||||
|
# if [[ -z "${JAVA_HOME}" ]]; then
|
||||||
|
# echo "ERROR: JAVA_HOME is not set and could not be found." 1>&2
|
||||||
|
# exit 1
|
||||||
|
# fi
|
||||||
|
#
|
||||||
|
# if [[ ! -d "${JAVA_HOME}" ]]; then
|
||||||
|
# echo "ERROR: JAVA_HOME (${JAVA_HOME}) does not exist." 1>&2
|
||||||
|
# exit 1
|
||||||
|
# fi
|
||||||
|
#
|
||||||
|
# JAVA="${JAVA_HOME}/bin/java"
|
||||||
|
#
|
||||||
|
# if [[ ! -x ${JAVA} ]]; then
|
||||||
|
# echo "ERROR: ${JAVA} is not executable." 1>&2
|
||||||
|
# exit 1
|
||||||
|
# fi
|
||||||
|
# JAVA_HEAP_MAX=-Xmx1g
|
||||||
|
# HADOOP_HEAPSIZE=${HADOOP_HEAPSIZE:-128}
|
||||||
|
#
|
||||||
|
# # check envvars which might override default args
|
||||||
|
# if [[ -n "$HADOOP_HEAPSIZE" ]]; then
|
||||||
|
# JAVA_HEAP_MAX="-Xmx${HADOOP_HEAPSIZE}m"
|
||||||
|
# fi
|
||||||
|
#}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,67 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.crypto;
|
||||||
|
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
|
|
||||||
|
import com.google.common.base.Preconditions;
|
||||||
|
|
||||||
|
@InterfaceAudience.Private
|
||||||
|
@InterfaceStability.Evolving
|
||||||
|
public abstract class AesCtrCryptoCodec extends CryptoCodec {
|
||||||
|
|
||||||
|
protected static final CipherSuite SUITE = CipherSuite.AES_CTR_NOPADDING;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* For AES, the algorithm block is fixed size of 128 bits.
|
||||||
|
* @see http://en.wikipedia.org/wiki/Advanced_Encryption_Standard
|
||||||
|
*/
|
||||||
|
private static final int AES_BLOCK_SIZE = SUITE.getAlgorithmBlockSize();
|
||||||
|
private static final int CTR_OFFSET = 8;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public CipherSuite getCipherSuite() {
|
||||||
|
return SUITE;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The IV is produced by adding the initial IV to the counter. IV length
|
||||||
|
* should be the same as {@link #AES_BLOCK_SIZE}
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public void calculateIV(byte[] initIV, long counter, byte[] IV) {
|
||||||
|
Preconditions.checkArgument(initIV.length == AES_BLOCK_SIZE);
|
||||||
|
Preconditions.checkArgument(IV.length == AES_BLOCK_SIZE);
|
||||||
|
|
||||||
|
System.arraycopy(initIV, 0, IV, 0, CTR_OFFSET);
|
||||||
|
long l = 0;
|
||||||
|
for (int i = 0; i < 8; i++) {
|
||||||
|
l = ((l << 8) | (initIV[CTR_OFFSET + i] & 0xff));
|
||||||
|
}
|
||||||
|
l += counter;
|
||||||
|
IV[CTR_OFFSET + 0] = (byte) (l >>> 56);
|
||||||
|
IV[CTR_OFFSET + 1] = (byte) (l >>> 48);
|
||||||
|
IV[CTR_OFFSET + 2] = (byte) (l >>> 40);
|
||||||
|
IV[CTR_OFFSET + 3] = (byte) (l >>> 32);
|
||||||
|
IV[CTR_OFFSET + 4] = (byte) (l >>> 24);
|
||||||
|
IV[CTR_OFFSET + 5] = (byte) (l >>> 16);
|
||||||
|
IV[CTR_OFFSET + 6] = (byte) (l >>> 8);
|
||||||
|
IV[CTR_OFFSET + 7] = (byte) (l);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,115 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.crypto;
|
||||||
|
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Defines properties of a CipherSuite. Modeled after the ciphers in
|
||||||
|
* {@link javax.crypto.Cipher}.
|
||||||
|
*/
|
||||||
|
@InterfaceAudience.Private
|
||||||
|
public enum CipherSuite {
|
||||||
|
UNKNOWN("Unknown", 0),
|
||||||
|
AES_CTR_NOPADDING("AES/CTR/NoPadding", 16);
|
||||||
|
|
||||||
|
private final String name;
|
||||||
|
private final int algoBlockSize;
|
||||||
|
|
||||||
|
private Integer unknownValue = null;
|
||||||
|
|
||||||
|
CipherSuite(String name, int algoBlockSize) {
|
||||||
|
this.name = name;
|
||||||
|
this.algoBlockSize = algoBlockSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setUnknownValue(int unknown) {
|
||||||
|
this.unknownValue = unknown;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getUnknownValue() {
|
||||||
|
return unknownValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return name of cipher suite, as in {@link javax.crypto.Cipher}
|
||||||
|
*/
|
||||||
|
public String getName() {
|
||||||
|
return name;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return size of an algorithm block in bytes
|
||||||
|
*/
|
||||||
|
public int getAlgorithmBlockSize() {
|
||||||
|
return algoBlockSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
StringBuilder builder = new StringBuilder("{");
|
||||||
|
builder.append("name: " + name);
|
||||||
|
builder.append(", algorithmBlockSize: " + algoBlockSize);
|
||||||
|
if (unknownValue != null) {
|
||||||
|
builder.append(", unknownValue: " + unknownValue);
|
||||||
|
}
|
||||||
|
builder.append("}");
|
||||||
|
return builder.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void checkName(String name) {
|
||||||
|
CipherSuite[] suites = CipherSuite.values();
|
||||||
|
for (CipherSuite suite : suites) {
|
||||||
|
if (suite.getName().equals(name)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new IllegalArgumentException("Invalid cipher suite name: " + name);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Convert to CipherSuite from name, {@link #algoBlockSize} is fixed for
|
||||||
|
* certain cipher suite, just need to compare the name.
|
||||||
|
* @param name cipher suite name
|
||||||
|
* @return CipherSuite cipher suite
|
||||||
|
*/
|
||||||
|
public static CipherSuite convert(String name) {
|
||||||
|
CipherSuite[] suites = CipherSuite.values();
|
||||||
|
for (CipherSuite suite : suites) {
|
||||||
|
if (suite.getName().equals(name)) {
|
||||||
|
return suite;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new IllegalArgumentException("Invalid cipher suite name: " + name);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns suffix of cipher suite configuration.
|
||||||
|
* @return String configuration suffix
|
||||||
|
*/
|
||||||
|
public String getConfigSuffix() {
|
||||||
|
String[] parts = name.split("/");
|
||||||
|
StringBuilder suffix = new StringBuilder();
|
||||||
|
for (String part : parts) {
|
||||||
|
suffix.append(".").append(part.toLowerCase());
|
||||||
|
}
|
||||||
|
|
||||||
|
return suffix.toString();
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,174 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.crypto;
|
||||||
|
|
||||||
|
import java.security.GeneralSecurityException;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
|
import org.apache.hadoop.conf.Configurable;
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.util.ReflectionUtils;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import com.google.common.base.Splitter;
|
||||||
|
import com.google.common.collect.Lists;
|
||||||
|
|
||||||
|
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX;
|
||||||
|
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY;
|
||||||
|
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_DEFAULT;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Crypto codec class, encapsulates encryptor/decryptor pair.
|
||||||
|
*/
|
||||||
|
@InterfaceAudience.Private
|
||||||
|
@InterfaceStability.Evolving
|
||||||
|
public abstract class CryptoCodec implements Configurable {
|
||||||
|
public static Logger LOG = LoggerFactory.getLogger(CryptoCodec.class);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get crypto codec for specified algorithm/mode/padding.
|
||||||
|
*
|
||||||
|
* @param conf
|
||||||
|
* the configuration
|
||||||
|
* @param CipherSuite
|
||||||
|
* algorithm/mode/padding
|
||||||
|
* @return CryptoCodec the codec object. Null value will be returned if no
|
||||||
|
* crypto codec classes with cipher suite configured.
|
||||||
|
*/
|
||||||
|
public static CryptoCodec getInstance(Configuration conf,
|
||||||
|
CipherSuite cipherSuite) {
|
||||||
|
List<Class<? extends CryptoCodec>> klasses = getCodecClasses(
|
||||||
|
conf, cipherSuite);
|
||||||
|
if (klasses == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
CryptoCodec codec = null;
|
||||||
|
for (Class<? extends CryptoCodec> klass : klasses) {
|
||||||
|
try {
|
||||||
|
CryptoCodec c = ReflectionUtils.newInstance(klass, conf);
|
||||||
|
if (c.getCipherSuite().getName().equals(cipherSuite.getName())) {
|
||||||
|
if (codec == null) {
|
||||||
|
LOG.debug("Using crypto codec {}.", klass.getName());
|
||||||
|
codec = c;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
LOG.warn("Crypto codec {} doesn't meet the cipher suite {}.",
|
||||||
|
klass.getName(), cipherSuite.getName());
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
LOG.warn("Crypto codec {} is not available.", klass.getName());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (codec != null) {
|
||||||
|
return codec;
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new RuntimeException("No available crypto codec which meets " +
|
||||||
|
"the cipher suite " + cipherSuite.getName() + ".");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get crypto codec for algorithm/mode/padding in config value
|
||||||
|
* hadoop.security.crypto.cipher.suite
|
||||||
|
*
|
||||||
|
* @param conf
|
||||||
|
* the configuration
|
||||||
|
* @return CryptoCodec the codec object Null value will be returned if no
|
||||||
|
* crypto codec classes with cipher suite configured.
|
||||||
|
*/
|
||||||
|
public static CryptoCodec getInstance(Configuration conf) {
|
||||||
|
String name = conf.get(HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY,
|
||||||
|
HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_DEFAULT);
|
||||||
|
return getInstance(conf, CipherSuite.convert(name));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static List<Class<? extends CryptoCodec>> getCodecClasses(
|
||||||
|
Configuration conf, CipherSuite cipherSuite) {
|
||||||
|
List<Class<? extends CryptoCodec>> result = Lists.newArrayList();
|
||||||
|
String configName = HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX +
|
||||||
|
cipherSuite.getConfigSuffix();
|
||||||
|
String codecString = conf.get(configName);
|
||||||
|
if (codecString == null) {
|
||||||
|
LOG.warn("No crypto codec classes with cipher suite configured.");
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
for (String c : Splitter.on(',').trimResults().omitEmptyStrings().
|
||||||
|
split(codecString)) {
|
||||||
|
try {
|
||||||
|
Class<?> cls = conf.getClassByName(c);
|
||||||
|
result.add(cls.asSubclass(CryptoCodec.class));
|
||||||
|
} catch (ClassCastException e) {
|
||||||
|
LOG.warn("Class " + c + " is not a CryptoCodec.");
|
||||||
|
} catch (ClassNotFoundException e) {
|
||||||
|
LOG.warn("Crypto codec " + c + " not found.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return the CipherSuite for this codec.
|
||||||
|
*/
|
||||||
|
public abstract CipherSuite getCipherSuite();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a {@link org.apache.hadoop.crypto.Encryptor}.
|
||||||
|
* @return Encryptor the encryptor
|
||||||
|
*/
|
||||||
|
public abstract Encryptor createEncryptor() throws GeneralSecurityException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a {@link org.apache.hadoop.crypto.Decryptor}.
|
||||||
|
* @return Decryptor the decryptor
|
||||||
|
*/
|
||||||
|
public abstract Decryptor createDecryptor() throws GeneralSecurityException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This interface is only for Counter (CTR) mode. Generally the Encryptor
|
||||||
|
* or Decryptor calculates the IV and maintain encryption context internally.
|
||||||
|
* For example a {@link javax.crypto.Cipher} will maintain its encryption
|
||||||
|
* context internally when we do encryption/decryption using the
|
||||||
|
* Cipher#update interface.
|
||||||
|
* <p/>
|
||||||
|
* Encryption/Decryption is not always on the entire file. For example,
|
||||||
|
* in Hadoop, a node may only decrypt a portion of a file (i.e. a split).
|
||||||
|
* In these situations, the counter is derived from the file position.
|
||||||
|
* <p/>
|
||||||
|
* The IV can be calculated by combining the initial IV and the counter with
|
||||||
|
* a lossless operation (concatenation, addition, or XOR).
|
||||||
|
* @see http://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Counter_.28CTR.29
|
||||||
|
*
|
||||||
|
* @param initIV initial IV
|
||||||
|
* @param counter counter for input stream position
|
||||||
|
* @param IV the IV for input stream position
|
||||||
|
*/
|
||||||
|
public abstract void calculateIV(byte[] initIV, long counter, byte[] IV);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generate a number of secure, random bytes suitable for cryptographic use.
|
||||||
|
* This method needs to be thread-safe.
|
||||||
|
*
|
||||||
|
* @param bytes byte array to populate with random data
|
||||||
|
*/
|
||||||
|
public abstract void generateSecureRandom(byte[] bytes);
|
||||||
|
}
|
|
@ -0,0 +1,680 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.crypto;
|
||||||
|
|
||||||
|
import java.io.FileDescriptor;
|
||||||
|
import java.io.FileInputStream;
|
||||||
|
import java.io.FilterInputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.nio.ByteBuffer;
|
||||||
|
import java.security.GeneralSecurityException;
|
||||||
|
import java.util.EnumSet;
|
||||||
|
import java.util.Queue;
|
||||||
|
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||||
|
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
|
import org.apache.hadoop.fs.ByteBufferReadable;
|
||||||
|
import org.apache.hadoop.fs.CanSetDropBehind;
|
||||||
|
import org.apache.hadoop.fs.CanSetReadahead;
|
||||||
|
import org.apache.hadoop.fs.HasEnhancedByteBufferAccess;
|
||||||
|
import org.apache.hadoop.fs.HasFileDescriptor;
|
||||||
|
import org.apache.hadoop.fs.PositionedReadable;
|
||||||
|
import org.apache.hadoop.fs.ReadOption;
|
||||||
|
import org.apache.hadoop.fs.Seekable;
|
||||||
|
import org.apache.hadoop.io.ByteBufferPool;
|
||||||
|
|
||||||
|
import com.google.common.base.Preconditions;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* CryptoInputStream decrypts data. It is not thread-safe. AES CTR mode is
|
||||||
|
* required in order to ensure that the plain text and cipher text have a 1:1
|
||||||
|
* mapping. The decryption is buffer based. The key points of the decryption
|
||||||
|
* are (1) calculating the counter and (2) padding through stream position:
|
||||||
|
* <p/>
|
||||||
|
* counter = base + pos/(algorithm blocksize);
|
||||||
|
* padding = pos%(algorithm blocksize);
|
||||||
|
* <p/>
|
||||||
|
* The underlying stream offset is maintained as state.
|
||||||
|
*/
|
||||||
|
@InterfaceAudience.Private
|
||||||
|
@InterfaceStability.Evolving
|
||||||
|
public class CryptoInputStream extends FilterInputStream implements
|
||||||
|
Seekable, PositionedReadable, ByteBufferReadable, HasFileDescriptor,
|
||||||
|
CanSetDropBehind, CanSetReadahead, HasEnhancedByteBufferAccess {
|
||||||
|
private static final byte[] oneByteBuf = new byte[1];
|
||||||
|
private final CryptoCodec codec;
|
||||||
|
private final Decryptor decryptor;
|
||||||
|
private final int bufferSize;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Input data buffer. The data starts at inBuffer.position() and ends at
|
||||||
|
* to inBuffer.limit().
|
||||||
|
*/
|
||||||
|
private ByteBuffer inBuffer;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The decrypted data buffer. The data starts at outBuffer.position() and
|
||||||
|
* ends at outBuffer.limit();
|
||||||
|
*/
|
||||||
|
private ByteBuffer outBuffer;
|
||||||
|
private long streamOffset = 0; // Underlying stream offset.
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Whether the underlying stream supports
|
||||||
|
* {@link org.apache.hadoop.fs.ByteBufferReadable}
|
||||||
|
*/
|
||||||
|
private Boolean usingByteBufferRead = null;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Padding = pos%(algorithm blocksize); Padding is put into {@link #inBuffer}
|
||||||
|
* before any other data goes in. The purpose of padding is to put the input
|
||||||
|
* data at proper position.
|
||||||
|
*/
|
||||||
|
private byte padding;
|
||||||
|
private boolean closed;
|
||||||
|
private final byte[] key;
|
||||||
|
private final byte[] initIV;
|
||||||
|
private byte[] iv;
|
||||||
|
|
||||||
|
/** DirectBuffer pool */
|
||||||
|
private final Queue<ByteBuffer> bufferPool =
|
||||||
|
new ConcurrentLinkedQueue<ByteBuffer>();
|
||||||
|
/** Decryptor pool */
|
||||||
|
private final Queue<Decryptor> decryptorPool =
|
||||||
|
new ConcurrentLinkedQueue<Decryptor>();
|
||||||
|
|
||||||
|
public CryptoInputStream(InputStream in, CryptoCodec codec,
|
||||||
|
int bufferSize, byte[] key, byte[] iv) throws IOException {
|
||||||
|
this(in, codec, bufferSize, key, iv,
|
||||||
|
CryptoStreamUtils.getInputStreamOffset(in));
|
||||||
|
}
|
||||||
|
|
||||||
|
public CryptoInputStream(InputStream in, CryptoCodec codec,
|
||||||
|
int bufferSize, byte[] key, byte[] iv, long streamOffset) throws IOException {
|
||||||
|
super(in);
|
||||||
|
this.bufferSize = CryptoStreamUtils.checkBufferSize(codec, bufferSize);
|
||||||
|
this.codec = codec;
|
||||||
|
this.key = key.clone();
|
||||||
|
this.initIV = iv.clone();
|
||||||
|
this.iv = iv.clone();
|
||||||
|
this.streamOffset = streamOffset;
|
||||||
|
inBuffer = ByteBuffer.allocateDirect(this.bufferSize);
|
||||||
|
outBuffer = ByteBuffer.allocateDirect(this.bufferSize);
|
||||||
|
decryptor = getDecryptor();
|
||||||
|
resetStreamOffset(streamOffset);
|
||||||
|
}
|
||||||
|
|
||||||
|
public CryptoInputStream(InputStream in, CryptoCodec codec,
|
||||||
|
byte[] key, byte[] iv) throws IOException {
|
||||||
|
this(in, codec, CryptoStreamUtils.getBufferSize(codec.getConf()), key, iv);
|
||||||
|
}
|
||||||
|
|
||||||
|
public InputStream getWrappedStream() {
|
||||||
|
return in;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decryption is buffer based.
|
||||||
|
* If there is data in {@link #outBuffer}, then read it out of this buffer.
|
||||||
|
* If there is no data in {@link #outBuffer}, then read more from the
|
||||||
|
* underlying stream and do the decryption.
|
||||||
|
* @param b the buffer into which the decrypted data is read.
|
||||||
|
* @param off the buffer offset.
|
||||||
|
* @param len the maximum number of decrypted data bytes to read.
|
||||||
|
* @return int the total number of decrypted data bytes read into the buffer.
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public int read(byte[] b, int off, int len) throws IOException {
|
||||||
|
checkStream();
|
||||||
|
if (b == null) {
|
||||||
|
throw new NullPointerException();
|
||||||
|
} else if (off < 0 || len < 0 || len > b.length - off) {
|
||||||
|
throw new IndexOutOfBoundsException();
|
||||||
|
} else if (len == 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
final int remaining = outBuffer.remaining();
|
||||||
|
if (remaining > 0) {
|
||||||
|
int n = Math.min(len, remaining);
|
||||||
|
outBuffer.get(b, off, n);
|
||||||
|
return n;
|
||||||
|
} else {
|
||||||
|
int n = 0;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Check whether the underlying stream is {@link ByteBufferReadable},
|
||||||
|
* it can avoid bytes copy.
|
||||||
|
*/
|
||||||
|
if (usingByteBufferRead == null) {
|
||||||
|
if (in instanceof ByteBufferReadable) {
|
||||||
|
try {
|
||||||
|
n = ((ByteBufferReadable) in).read(inBuffer);
|
||||||
|
usingByteBufferRead = Boolean.TRUE;
|
||||||
|
} catch (UnsupportedOperationException e) {
|
||||||
|
usingByteBufferRead = Boolean.FALSE;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
usingByteBufferRead = Boolean.FALSE;
|
||||||
|
}
|
||||||
|
if (!usingByteBufferRead) {
|
||||||
|
n = readFromUnderlyingStream(inBuffer);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (usingByteBufferRead) {
|
||||||
|
n = ((ByteBufferReadable) in).read(inBuffer);
|
||||||
|
} else {
|
||||||
|
n = readFromUnderlyingStream(inBuffer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (n <= 0) {
|
||||||
|
return n;
|
||||||
|
}
|
||||||
|
|
||||||
|
streamOffset += n; // Read n bytes
|
||||||
|
decrypt(decryptor, inBuffer, outBuffer, padding);
|
||||||
|
padding = afterDecryption(decryptor, inBuffer, streamOffset, iv);
|
||||||
|
n = Math.min(len, outBuffer.remaining());
|
||||||
|
outBuffer.get(b, off, n);
|
||||||
|
return n;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Read data from underlying stream. */
|
||||||
|
private int readFromUnderlyingStream(ByteBuffer inBuffer) throws IOException {
|
||||||
|
final int toRead = inBuffer.remaining();
|
||||||
|
final byte[] tmp = getTmpBuf();
|
||||||
|
final int n = in.read(tmp, 0, toRead);
|
||||||
|
if (n > 0) {
|
||||||
|
inBuffer.put(tmp, 0, n);
|
||||||
|
}
|
||||||
|
return n;
|
||||||
|
}
|
||||||
|
|
||||||
|
private byte[] tmpBuf;
|
||||||
|
private byte[] getTmpBuf() {
|
||||||
|
if (tmpBuf == null) {
|
||||||
|
tmpBuf = new byte[bufferSize];
|
||||||
|
}
|
||||||
|
return tmpBuf;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Do the decryption using inBuffer as input and outBuffer as output.
|
||||||
|
* Upon return, inBuffer is cleared; the decrypted data starts at
|
||||||
|
* outBuffer.position() and ends at outBuffer.limit();
|
||||||
|
*/
|
||||||
|
private void decrypt(Decryptor decryptor, ByteBuffer inBuffer,
|
||||||
|
ByteBuffer outBuffer, byte padding) throws IOException {
|
||||||
|
Preconditions.checkState(inBuffer.position() >= padding);
|
||||||
|
if(inBuffer.position() == padding) {
|
||||||
|
// There is no real data in inBuffer.
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
inBuffer.flip();
|
||||||
|
outBuffer.clear();
|
||||||
|
decryptor.decrypt(inBuffer, outBuffer);
|
||||||
|
inBuffer.clear();
|
||||||
|
outBuffer.flip();
|
||||||
|
if (padding > 0) {
|
||||||
|
/*
|
||||||
|
* The plain text and cipher text have a 1:1 mapping, they start at the
|
||||||
|
* same position.
|
||||||
|
*/
|
||||||
|
outBuffer.position(padding);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This method is executed immediately after decryption. Check whether
|
||||||
|
* decryptor should be updated and recalculate padding if needed.
|
||||||
|
*/
|
||||||
|
private byte afterDecryption(Decryptor decryptor, ByteBuffer inBuffer,
|
||||||
|
long position, byte[] iv) throws IOException {
|
||||||
|
byte padding = 0;
|
||||||
|
if (decryptor.isContextReset()) {
|
||||||
|
/*
|
||||||
|
* This code is generally not executed since the decryptor usually
|
||||||
|
* maintains decryption context (e.g. the counter) internally. However,
|
||||||
|
* some implementations can't maintain context so a re-init is necessary
|
||||||
|
* after each decryption call.
|
||||||
|
*/
|
||||||
|
updateDecryptor(decryptor, position, iv);
|
||||||
|
padding = getPadding(position);
|
||||||
|
inBuffer.position(padding);
|
||||||
|
}
|
||||||
|
return padding;
|
||||||
|
}
|
||||||
|
|
||||||
|
private long getCounter(long position) {
|
||||||
|
return position / codec.getCipherSuite().getAlgorithmBlockSize();
|
||||||
|
}
|
||||||
|
|
||||||
|
private byte getPadding(long position) {
|
||||||
|
return (byte)(position % codec.getCipherSuite().getAlgorithmBlockSize());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Calculate the counter and iv, update the decryptor. */
|
||||||
|
private void updateDecryptor(Decryptor decryptor, long position, byte[] iv)
|
||||||
|
throws IOException {
|
||||||
|
final long counter = getCounter(position);
|
||||||
|
codec.calculateIV(initIV, counter, iv);
|
||||||
|
decryptor.init(key, iv);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reset the underlying stream offset; clear {@link #inBuffer} and
|
||||||
|
* {@link #outBuffer}. This Typically happens during {@link #seek(long)}
|
||||||
|
* or {@link #skip(long)}.
|
||||||
|
*/
|
||||||
|
private void resetStreamOffset(long offset) throws IOException {
|
||||||
|
streamOffset = offset;
|
||||||
|
inBuffer.clear();
|
||||||
|
outBuffer.clear();
|
||||||
|
outBuffer.limit(0);
|
||||||
|
updateDecryptor(decryptor, offset, iv);
|
||||||
|
padding = getPadding(offset);
|
||||||
|
inBuffer.position(padding); // Set proper position for input data.
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() throws IOException {
|
||||||
|
if (closed) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
super.close();
|
||||||
|
freeBuffers();
|
||||||
|
closed = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Positioned read. It is thread-safe */
|
||||||
|
@Override
|
||||||
|
public int read(long position, byte[] buffer, int offset, int length)
|
||||||
|
throws IOException {
|
||||||
|
checkStream();
|
||||||
|
try {
|
||||||
|
final int n = ((PositionedReadable) in).read(position, buffer, offset,
|
||||||
|
length);
|
||||||
|
if (n > 0) {
|
||||||
|
// This operation does not change the current offset of the file
|
||||||
|
decrypt(position, buffer, offset, n);
|
||||||
|
}
|
||||||
|
|
||||||
|
return n;
|
||||||
|
} catch (ClassCastException e) {
|
||||||
|
throw new UnsupportedOperationException("This stream does not support " +
|
||||||
|
"positioned read.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decrypt length bytes in buffer starting at offset. Output is also put
|
||||||
|
* into buffer starting at offset. It is thread-safe.
|
||||||
|
*/
|
||||||
|
private void decrypt(long position, byte[] buffer, int offset, int length)
|
||||||
|
throws IOException {
|
||||||
|
ByteBuffer inBuffer = getBuffer();
|
||||||
|
ByteBuffer outBuffer = getBuffer();
|
||||||
|
Decryptor decryptor = null;
|
||||||
|
try {
|
||||||
|
decryptor = getDecryptor();
|
||||||
|
byte[] iv = initIV.clone();
|
||||||
|
updateDecryptor(decryptor, position, iv);
|
||||||
|
byte padding = getPadding(position);
|
||||||
|
inBuffer.position(padding); // Set proper position for input data.
|
||||||
|
|
||||||
|
int n = 0;
|
||||||
|
while (n < length) {
|
||||||
|
int toDecrypt = Math.min(length - n, inBuffer.remaining());
|
||||||
|
inBuffer.put(buffer, offset + n, toDecrypt);
|
||||||
|
// Do decryption
|
||||||
|
decrypt(decryptor, inBuffer, outBuffer, padding);
|
||||||
|
|
||||||
|
outBuffer.get(buffer, offset + n, toDecrypt);
|
||||||
|
n += toDecrypt;
|
||||||
|
padding = afterDecryption(decryptor, inBuffer, position + n, iv);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
returnBuffer(inBuffer);
|
||||||
|
returnBuffer(outBuffer);
|
||||||
|
returnDecryptor(decryptor);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Positioned read fully. It is thread-safe */
|
||||||
|
@Override
|
||||||
|
public void readFully(long position, byte[] buffer, int offset, int length)
|
||||||
|
throws IOException {
|
||||||
|
checkStream();
|
||||||
|
try {
|
||||||
|
((PositionedReadable) in).readFully(position, buffer, offset, length);
|
||||||
|
if (length > 0) {
|
||||||
|
// This operation does not change the current offset of the file
|
||||||
|
decrypt(position, buffer, offset, length);
|
||||||
|
}
|
||||||
|
} catch (ClassCastException e) {
|
||||||
|
throw new UnsupportedOperationException("This stream does not support " +
|
||||||
|
"positioned readFully.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void readFully(long position, byte[] buffer) throws IOException {
|
||||||
|
readFully(position, buffer, 0, buffer.length);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Seek to a position. */
|
||||||
|
@Override
|
||||||
|
public void seek(long pos) throws IOException {
|
||||||
|
Preconditions.checkArgument(pos >= 0, "Cannot seek to negative offset.");
|
||||||
|
checkStream();
|
||||||
|
try {
|
||||||
|
/*
|
||||||
|
* If data of target pos in the underlying stream has already been read
|
||||||
|
* and decrypted in outBuffer, we just need to re-position outBuffer.
|
||||||
|
*/
|
||||||
|
if (pos <= streamOffset && pos >= (streamOffset - outBuffer.remaining())) {
|
||||||
|
int forward = (int) (pos - (streamOffset - outBuffer.remaining()));
|
||||||
|
if (forward > 0) {
|
||||||
|
outBuffer.position(outBuffer.position() + forward);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
((Seekable) in).seek(pos);
|
||||||
|
resetStreamOffset(pos);
|
||||||
|
}
|
||||||
|
} catch (ClassCastException e) {
|
||||||
|
throw new UnsupportedOperationException("This stream does not support " +
|
||||||
|
"seek.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Skip n bytes */
|
||||||
|
@Override
|
||||||
|
public long skip(long n) throws IOException {
|
||||||
|
Preconditions.checkArgument(n >= 0, "Negative skip length.");
|
||||||
|
checkStream();
|
||||||
|
|
||||||
|
if (n == 0) {
|
||||||
|
return 0;
|
||||||
|
} else if (n <= outBuffer.remaining()) {
|
||||||
|
int pos = outBuffer.position() + (int) n;
|
||||||
|
outBuffer.position(pos);
|
||||||
|
return n;
|
||||||
|
} else {
|
||||||
|
/*
|
||||||
|
* Subtract outBuffer.remaining() to see how many bytes we need to
|
||||||
|
* skip in the underlying stream. Add outBuffer.remaining() to the
|
||||||
|
* actual number of skipped bytes in the underlying stream to get the
|
||||||
|
* number of skipped bytes from the user's point of view.
|
||||||
|
*/
|
||||||
|
n -= outBuffer.remaining();
|
||||||
|
long skipped = in.skip(n);
|
||||||
|
if (skipped < 0) {
|
||||||
|
skipped = 0;
|
||||||
|
}
|
||||||
|
long pos = streamOffset + skipped;
|
||||||
|
skipped += outBuffer.remaining();
|
||||||
|
resetStreamOffset(pos);
|
||||||
|
return skipped;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Get underlying stream position. */
|
||||||
|
@Override
|
||||||
|
public long getPos() throws IOException {
|
||||||
|
checkStream();
|
||||||
|
// Equals: ((Seekable) in).getPos() - outBuffer.remaining()
|
||||||
|
return streamOffset - outBuffer.remaining();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** ByteBuffer read. */
|
||||||
|
@Override
|
||||||
|
public int read(ByteBuffer buf) throws IOException {
|
||||||
|
checkStream();
|
||||||
|
if (in instanceof ByteBufferReadable) {
|
||||||
|
final int unread = outBuffer.remaining();
|
||||||
|
if (unread > 0) { // Have unread decrypted data in buffer.
|
||||||
|
int toRead = buf.remaining();
|
||||||
|
if (toRead <= unread) {
|
||||||
|
final int limit = outBuffer.limit();
|
||||||
|
outBuffer.limit(outBuffer.position() + toRead);
|
||||||
|
buf.put(outBuffer);
|
||||||
|
outBuffer.limit(limit);
|
||||||
|
return toRead;
|
||||||
|
} else {
|
||||||
|
buf.put(outBuffer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
final int pos = buf.position();
|
||||||
|
final int n = ((ByteBufferReadable) in).read(buf);
|
||||||
|
if (n > 0) {
|
||||||
|
streamOffset += n; // Read n bytes
|
||||||
|
decrypt(buf, n, pos);
|
||||||
|
}
|
||||||
|
return n;
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new UnsupportedOperationException("ByteBuffer read unsupported " +
|
||||||
|
"by input stream.");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decrypt all data in buf: total n bytes from given start position.
|
||||||
|
* Output is also buf and same start position.
|
||||||
|
* buf.position() and buf.limit() should be unchanged after decryption.
|
||||||
|
*/
|
||||||
|
private void decrypt(ByteBuffer buf, int n, int start)
|
||||||
|
throws IOException {
|
||||||
|
final int pos = buf.position();
|
||||||
|
final int limit = buf.limit();
|
||||||
|
int len = 0;
|
||||||
|
while (len < n) {
|
||||||
|
buf.position(start + len);
|
||||||
|
buf.limit(start + len + Math.min(n - len, inBuffer.remaining()));
|
||||||
|
inBuffer.put(buf);
|
||||||
|
// Do decryption
|
||||||
|
try {
|
||||||
|
decrypt(decryptor, inBuffer, outBuffer, padding);
|
||||||
|
buf.position(start + len);
|
||||||
|
buf.limit(limit);
|
||||||
|
len += outBuffer.remaining();
|
||||||
|
buf.put(outBuffer);
|
||||||
|
} finally {
|
||||||
|
padding = afterDecryption(decryptor, inBuffer, streamOffset - (n - len), iv);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
buf.position(pos);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int available() throws IOException {
|
||||||
|
checkStream();
|
||||||
|
|
||||||
|
return in.available() + outBuffer.remaining();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean markSupported() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void mark(int readLimit) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void reset() throws IOException {
|
||||||
|
throw new IOException("Mark/reset not supported");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean seekToNewSource(long targetPos) throws IOException {
|
||||||
|
Preconditions.checkArgument(targetPos >= 0,
|
||||||
|
"Cannot seek to negative offset.");
|
||||||
|
checkStream();
|
||||||
|
try {
|
||||||
|
boolean result = ((Seekable) in).seekToNewSource(targetPos);
|
||||||
|
resetStreamOffset(targetPos);
|
||||||
|
return result;
|
||||||
|
} catch (ClassCastException e) {
|
||||||
|
throw new UnsupportedOperationException("This stream does not support " +
|
||||||
|
"seekToNewSource.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ByteBuffer read(ByteBufferPool bufferPool, int maxLength,
|
||||||
|
EnumSet<ReadOption> opts) throws IOException,
|
||||||
|
UnsupportedOperationException {
|
||||||
|
checkStream();
|
||||||
|
try {
|
||||||
|
if (outBuffer.remaining() > 0) {
|
||||||
|
// Have some decrypted data unread, need to reset.
|
||||||
|
((Seekable) in).seek(getPos());
|
||||||
|
resetStreamOffset(getPos());
|
||||||
|
}
|
||||||
|
final ByteBuffer buffer = ((HasEnhancedByteBufferAccess) in).
|
||||||
|
read(bufferPool, maxLength, opts);
|
||||||
|
if (buffer != null) {
|
||||||
|
final int n = buffer.remaining();
|
||||||
|
if (n > 0) {
|
||||||
|
streamOffset += buffer.remaining(); // Read n bytes
|
||||||
|
final int pos = buffer.position();
|
||||||
|
decrypt(buffer, n, pos);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return buffer;
|
||||||
|
} catch (ClassCastException e) {
|
||||||
|
throw new UnsupportedOperationException("This stream does not support " +
|
||||||
|
"enhanced byte buffer access.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void releaseBuffer(ByteBuffer buffer) {
|
||||||
|
try {
|
||||||
|
((HasEnhancedByteBufferAccess) in).releaseBuffer(buffer);
|
||||||
|
} catch (ClassCastException e) {
|
||||||
|
throw new UnsupportedOperationException("This stream does not support " +
|
||||||
|
"release buffer.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setReadahead(Long readahead) throws IOException,
|
||||||
|
UnsupportedOperationException {
|
||||||
|
try {
|
||||||
|
((CanSetReadahead) in).setReadahead(readahead);
|
||||||
|
} catch (ClassCastException e) {
|
||||||
|
throw new UnsupportedOperationException("This stream does not support " +
|
||||||
|
"setting the readahead caching strategy.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setDropBehind(Boolean dropCache) throws IOException,
|
||||||
|
UnsupportedOperationException {
|
||||||
|
try {
|
||||||
|
((CanSetDropBehind) in).setDropBehind(dropCache);
|
||||||
|
} catch (ClassCastException e) {
|
||||||
|
throw new UnsupportedOperationException("This stream does not " +
|
||||||
|
"support setting the drop-behind caching setting.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public FileDescriptor getFileDescriptor() throws IOException {
|
||||||
|
if (in instanceof HasFileDescriptor) {
|
||||||
|
return ((HasFileDescriptor) in).getFileDescriptor();
|
||||||
|
} else if (in instanceof FileInputStream) {
|
||||||
|
return ((FileInputStream) in).getFD();
|
||||||
|
} else {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int read() throws IOException {
|
||||||
|
return (read(oneByteBuf, 0, 1) == -1) ? -1 : (oneByteBuf[0] & 0xff);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void checkStream() throws IOException {
|
||||||
|
if (closed) {
|
||||||
|
throw new IOException("Stream closed");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Get direct buffer from pool */
|
||||||
|
private ByteBuffer getBuffer() {
|
||||||
|
ByteBuffer buffer = bufferPool.poll();
|
||||||
|
if (buffer == null) {
|
||||||
|
buffer = ByteBuffer.allocateDirect(bufferSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
return buffer;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Return direct buffer to pool */
|
||||||
|
private void returnBuffer(ByteBuffer buf) {
|
||||||
|
if (buf != null) {
|
||||||
|
buf.clear();
|
||||||
|
bufferPool.add(buf);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Forcibly free the direct buffers. */
|
||||||
|
private void freeBuffers() {
|
||||||
|
CryptoStreamUtils.freeDB(inBuffer);
|
||||||
|
CryptoStreamUtils.freeDB(outBuffer);
|
||||||
|
cleanBufferPool();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Clean direct buffer pool */
|
||||||
|
private void cleanBufferPool() {
|
||||||
|
ByteBuffer buf;
|
||||||
|
while ((buf = bufferPool.poll()) != null) {
|
||||||
|
CryptoStreamUtils.freeDB(buf);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Get decryptor from pool */
|
||||||
|
private Decryptor getDecryptor() throws IOException {
|
||||||
|
Decryptor decryptor = decryptorPool.poll();
|
||||||
|
if (decryptor == null) {
|
||||||
|
try {
|
||||||
|
decryptor = codec.createDecryptor();
|
||||||
|
} catch (GeneralSecurityException e) {
|
||||||
|
throw new IOException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return decryptor;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Return decryptor to pool */
|
||||||
|
private void returnDecryptor(Decryptor decryptor) {
|
||||||
|
if (decryptor != null) {
|
||||||
|
decryptorPool.add(decryptor);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,280 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.crypto;
|
||||||
|
|
||||||
|
import java.io.FilterOutputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.OutputStream;
|
||||||
|
import java.nio.ByteBuffer;
|
||||||
|
import java.security.GeneralSecurityException;
|
||||||
|
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
|
import org.apache.hadoop.fs.CanSetDropBehind;
|
||||||
|
import org.apache.hadoop.fs.Syncable;
|
||||||
|
|
||||||
|
import com.google.common.base.Preconditions;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* CryptoOutputStream encrypts data. It is not thread-safe. AES CTR mode is
|
||||||
|
* required in order to ensure that the plain text and cipher text have a 1:1
|
||||||
|
* mapping. The encryption is buffer based. The key points of the encryption are
|
||||||
|
* (1) calculating counter and (2) padding through stream position.
|
||||||
|
* <p/>
|
||||||
|
* counter = base + pos/(algorithm blocksize);
|
||||||
|
* padding = pos%(algorithm blocksize);
|
||||||
|
* <p/>
|
||||||
|
* The underlying stream offset is maintained as state.
|
||||||
|
*/
|
||||||
|
@InterfaceAudience.Private
|
||||||
|
@InterfaceStability.Evolving
|
||||||
|
public class CryptoOutputStream extends FilterOutputStream implements
|
||||||
|
Syncable, CanSetDropBehind {
|
||||||
|
private static final byte[] oneByteBuf = new byte[1];
|
||||||
|
private final CryptoCodec codec;
|
||||||
|
private final Encryptor encryptor;
|
||||||
|
private final int bufferSize;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Input data buffer. The data starts at inBuffer.position() and ends at
|
||||||
|
* inBuffer.limit().
|
||||||
|
*/
|
||||||
|
private ByteBuffer inBuffer;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Encrypted data buffer. The data starts at outBuffer.position() and ends at
|
||||||
|
* outBuffer.limit();
|
||||||
|
*/
|
||||||
|
private ByteBuffer outBuffer;
|
||||||
|
private long streamOffset = 0; // Underlying stream offset.
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Padding = pos%(algorithm blocksize); Padding is put into {@link #inBuffer}
|
||||||
|
* before any other data goes in. The purpose of padding is to put input data
|
||||||
|
* at proper position.
|
||||||
|
*/
|
||||||
|
private byte padding;
|
||||||
|
private boolean closed;
|
||||||
|
private final byte[] key;
|
||||||
|
private final byte[] initIV;
|
||||||
|
private byte[] iv;
|
||||||
|
|
||||||
|
public CryptoOutputStream(OutputStream out, CryptoCodec codec,
|
||||||
|
int bufferSize, byte[] key, byte[] iv) throws IOException {
|
||||||
|
this(out, codec, bufferSize, key, iv, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
public CryptoOutputStream(OutputStream out, CryptoCodec codec,
|
||||||
|
int bufferSize, byte[] key, byte[] iv, long streamOffset)
|
||||||
|
throws IOException {
|
||||||
|
super(out);
|
||||||
|
this.bufferSize = CryptoStreamUtils.checkBufferSize(codec, bufferSize);
|
||||||
|
this.codec = codec;
|
||||||
|
this.key = key.clone();
|
||||||
|
this.initIV = iv.clone();
|
||||||
|
this.iv = iv.clone();
|
||||||
|
inBuffer = ByteBuffer.allocateDirect(this.bufferSize);
|
||||||
|
outBuffer = ByteBuffer.allocateDirect(this.bufferSize);
|
||||||
|
this.streamOffset = streamOffset;
|
||||||
|
try {
|
||||||
|
encryptor = codec.createEncryptor();
|
||||||
|
} catch (GeneralSecurityException e) {
|
||||||
|
throw new IOException(e);
|
||||||
|
}
|
||||||
|
updateEncryptor();
|
||||||
|
}
|
||||||
|
|
||||||
|
public CryptoOutputStream(OutputStream out, CryptoCodec codec,
|
||||||
|
byte[] key, byte[] iv) throws IOException {
|
||||||
|
this(out, codec, key, iv, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
public CryptoOutputStream(OutputStream out, CryptoCodec codec,
|
||||||
|
byte[] key, byte[] iv, long streamOffset) throws IOException {
|
||||||
|
this(out, codec, CryptoStreamUtils.getBufferSize(codec.getConf()),
|
||||||
|
key, iv, streamOffset);
|
||||||
|
}
|
||||||
|
|
||||||
|
public OutputStream getWrappedStream() {
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Encryption is buffer based.
|
||||||
|
* If there is enough room in {@link #inBuffer}, then write to this buffer.
|
||||||
|
* If {@link #inBuffer} is full, then do encryption and write data to the
|
||||||
|
* underlying stream.
|
||||||
|
* @param b the data.
|
||||||
|
* @param off the start offset in the data.
|
||||||
|
* @param len the number of bytes to write.
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public void write(byte[] b, int off, int len) throws IOException {
|
||||||
|
checkStream();
|
||||||
|
if (b == null) {
|
||||||
|
throw new NullPointerException();
|
||||||
|
} else if (off < 0 || len < 0 || off > b.length ||
|
||||||
|
len > b.length - off) {
|
||||||
|
throw new IndexOutOfBoundsException();
|
||||||
|
}
|
||||||
|
while (len > 0) {
|
||||||
|
final int remaining = inBuffer.remaining();
|
||||||
|
if (len < remaining) {
|
||||||
|
inBuffer.put(b, off, len);
|
||||||
|
len = 0;
|
||||||
|
} else {
|
||||||
|
inBuffer.put(b, off, remaining);
|
||||||
|
off += remaining;
|
||||||
|
len -= remaining;
|
||||||
|
encrypt();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Do the encryption, input is {@link #inBuffer} and output is
|
||||||
|
* {@link #outBuffer}.
|
||||||
|
*/
|
||||||
|
private void encrypt() throws IOException {
|
||||||
|
Preconditions.checkState(inBuffer.position() >= padding);
|
||||||
|
if (inBuffer.position() == padding) {
|
||||||
|
// There is no real data in the inBuffer.
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
inBuffer.flip();
|
||||||
|
outBuffer.clear();
|
||||||
|
encryptor.encrypt(inBuffer, outBuffer);
|
||||||
|
inBuffer.clear();
|
||||||
|
outBuffer.flip();
|
||||||
|
if (padding > 0) {
|
||||||
|
/*
|
||||||
|
* The plain text and cipher text have a 1:1 mapping, they start at the
|
||||||
|
* same position.
|
||||||
|
*/
|
||||||
|
outBuffer.position(padding);
|
||||||
|
padding = 0;
|
||||||
|
}
|
||||||
|
final int len = outBuffer.remaining();
|
||||||
|
|
||||||
|
/*
|
||||||
|
* If underlying stream supports {@link ByteBuffer} write in future, needs
|
||||||
|
* refine here.
|
||||||
|
*/
|
||||||
|
final byte[] tmp = getTmpBuf();
|
||||||
|
outBuffer.get(tmp, 0, len);
|
||||||
|
out.write(tmp, 0, len);
|
||||||
|
|
||||||
|
streamOffset += len;
|
||||||
|
if (encryptor.isContextReset()) {
|
||||||
|
/*
|
||||||
|
* This code is generally not executed since the encryptor usually
|
||||||
|
* maintains encryption context (e.g. the counter) internally. However,
|
||||||
|
* some implementations can't maintain context so a re-init is necessary
|
||||||
|
* after each encryption call.
|
||||||
|
*/
|
||||||
|
updateEncryptor();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Update the {@link #encryptor}: calculate counter and {@link #padding}. */
|
||||||
|
private void updateEncryptor() throws IOException {
|
||||||
|
final long counter =
|
||||||
|
streamOffset / codec.getCipherSuite().getAlgorithmBlockSize();
|
||||||
|
padding =
|
||||||
|
(byte)(streamOffset % codec.getCipherSuite().getAlgorithmBlockSize());
|
||||||
|
inBuffer.position(padding); // Set proper position for input data.
|
||||||
|
codec.calculateIV(initIV, counter, iv);
|
||||||
|
encryptor.init(key, iv);
|
||||||
|
}
|
||||||
|
|
||||||
|
private byte[] tmpBuf;
|
||||||
|
private byte[] getTmpBuf() {
|
||||||
|
if (tmpBuf == null) {
|
||||||
|
tmpBuf = new byte[bufferSize];
|
||||||
|
}
|
||||||
|
return tmpBuf;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() throws IOException {
|
||||||
|
if (closed) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
super.close();
|
||||||
|
freeBuffers();
|
||||||
|
closed = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* To flush, we need to encrypt the data in the buffer and write to the
|
||||||
|
* underlying stream, then do the flush.
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public void flush() throws IOException {
|
||||||
|
checkStream();
|
||||||
|
encrypt();
|
||||||
|
super.flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void write(int b) throws IOException {
|
||||||
|
oneByteBuf[0] = (byte)(b & 0xff);
|
||||||
|
write(oneByteBuf, 0, oneByteBuf.length);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void checkStream() throws IOException {
|
||||||
|
if (closed) {
|
||||||
|
throw new IOException("Stream closed");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setDropBehind(Boolean dropCache) throws IOException,
|
||||||
|
UnsupportedOperationException {
|
||||||
|
try {
|
||||||
|
((CanSetDropBehind) out).setDropBehind(dropCache);
|
||||||
|
} catch (ClassCastException e) {
|
||||||
|
throw new UnsupportedOperationException("This stream does not " +
|
||||||
|
"support setting the drop-behind caching.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void hflush() throws IOException {
|
||||||
|
flush();
|
||||||
|
if (out instanceof Syncable) {
|
||||||
|
((Syncable)out).hflush();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void hsync() throws IOException {
|
||||||
|
flush();
|
||||||
|
if (out instanceof Syncable) {
|
||||||
|
((Syncable)out).hsync();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Forcibly free the direct buffers. */
|
||||||
|
private void freeBuffers() {
|
||||||
|
CryptoStreamUtils.freeDB(inBuffer);
|
||||||
|
CryptoStreamUtils.freeDB(outBuffer);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,70 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.crypto;
|
||||||
|
|
||||||
|
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_DEFAULT;
|
||||||
|
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_KEY;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.nio.ByteBuffer;
|
||||||
|
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.Seekable;
|
||||||
|
|
||||||
|
import com.google.common.base.Preconditions;
|
||||||
|
|
||||||
|
@InterfaceAudience.Private
|
||||||
|
public class CryptoStreamUtils {
|
||||||
|
private static final int MIN_BUFFER_SIZE = 512;
|
||||||
|
|
||||||
|
/** Forcibly free the direct buffer. */
|
||||||
|
public static void freeDB(ByteBuffer buffer) {
|
||||||
|
if (buffer instanceof sun.nio.ch.DirectBuffer) {
|
||||||
|
final sun.misc.Cleaner bufferCleaner =
|
||||||
|
((sun.nio.ch.DirectBuffer) buffer).cleaner();
|
||||||
|
bufferCleaner.clean();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Read crypto buffer size */
|
||||||
|
public static int getBufferSize(Configuration conf) {
|
||||||
|
return conf.getInt(HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_KEY,
|
||||||
|
HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_DEFAULT);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Check and floor buffer size */
|
||||||
|
public static int checkBufferSize(CryptoCodec codec, int bufferSize) {
|
||||||
|
Preconditions.checkArgument(bufferSize >= MIN_BUFFER_SIZE,
|
||||||
|
"Minimum value of buffer size is " + MIN_BUFFER_SIZE + ".");
|
||||||
|
return bufferSize - bufferSize % codec.getCipherSuite()
|
||||||
|
.getAlgorithmBlockSize();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* If input stream is {@link org.apache.hadoop.fs.Seekable}, return it's
|
||||||
|
* current position, otherwise return 0;
|
||||||
|
*/
|
||||||
|
public static long getInputStreamOffset(InputStream in) throws IOException {
|
||||||
|
if (in instanceof Seekable) {
|
||||||
|
return ((Seekable) in).getPos();
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,72 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.crypto;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.ByteBuffer;
|
||||||
|
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
|
|
||||||
|
@InterfaceAudience.Private
|
||||||
|
@InterfaceStability.Evolving
|
||||||
|
public interface Decryptor {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Initialize the decryptor and the internal decryption context.
|
||||||
|
* reset.
|
||||||
|
* @param key decryption key.
|
||||||
|
* @param iv decryption initialization vector
|
||||||
|
* @throws IOException if initialization fails
|
||||||
|
*/
|
||||||
|
public void init(byte[] key, byte[] iv) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Indicate whether the decryption context is reset.
|
||||||
|
* <p/>
|
||||||
|
* Certain modes, like CTR, require a different IV depending on the
|
||||||
|
* position in the stream. Generally, the decryptor maintains any necessary
|
||||||
|
* context for calculating the IV and counter so that no reinit is necessary
|
||||||
|
* during the decryption. Reinit before each operation is inefficient.
|
||||||
|
* @return boolean whether context is reset.
|
||||||
|
*/
|
||||||
|
public boolean isContextReset();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This presents a direct interface decrypting with direct ByteBuffers.
|
||||||
|
* <p/>
|
||||||
|
* This function does not always decrypt the entire buffer and may potentially
|
||||||
|
* need to be called multiple times to process an entire buffer. The object
|
||||||
|
* may hold the decryption context internally.
|
||||||
|
* <p/>
|
||||||
|
* Some implementations may require sufficient space in the destination
|
||||||
|
* buffer to decrypt the entire input buffer.
|
||||||
|
* <p/>
|
||||||
|
* Upon return, inBuffer.position() will be advanced by the number of bytes
|
||||||
|
* read and outBuffer.position() by bytes written. Implementations should
|
||||||
|
* not modify inBuffer.limit() and outBuffer.limit().
|
||||||
|
* <p/>
|
||||||
|
* @param inBuffer a direct {@link ByteBuffer} to read from. inBuffer may
|
||||||
|
* not be null and inBuffer.remaining() must be > 0
|
||||||
|
* @param outBuffer a direct {@link ByteBuffer} to write to. outBuffer may
|
||||||
|
* not be null and outBuffer.remaining() must be > 0
|
||||||
|
* @throws IOException if decryption fails
|
||||||
|
*/
|
||||||
|
public void decrypt(ByteBuffer inBuffer, ByteBuffer outBuffer)
|
||||||
|
throws IOException;
|
||||||
|
}
|
|
@ -0,0 +1,71 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.crypto;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.ByteBuffer;
|
||||||
|
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
|
|
||||||
|
@InterfaceAudience.Private
|
||||||
|
@InterfaceStability.Evolving
|
||||||
|
public interface Encryptor {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Initialize the encryptor and the internal encryption context.
|
||||||
|
* @param key encryption key.
|
||||||
|
* @param iv encryption initialization vector
|
||||||
|
* @throws IOException if initialization fails
|
||||||
|
*/
|
||||||
|
public void init(byte[] key, byte[] iv) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Indicate whether the encryption context is reset.
|
||||||
|
* <p/>
|
||||||
|
* Certain modes, like CTR, require a different IV depending on the
|
||||||
|
* position in the stream. Generally, the encryptor maintains any necessary
|
||||||
|
* context for calculating the IV and counter so that no reinit is necessary
|
||||||
|
* during the encryption. Reinit before each operation is inefficient.
|
||||||
|
* @return boolean whether context is reset.
|
||||||
|
*/
|
||||||
|
public boolean isContextReset();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This presents a direct interface encrypting with direct ByteBuffers.
|
||||||
|
* <p/>
|
||||||
|
* This function does not always encrypt the entire buffer and may potentially
|
||||||
|
* need to be called multiple times to process an entire buffer. The object
|
||||||
|
* may hold the encryption context internally.
|
||||||
|
* <p/>
|
||||||
|
* Some implementations may require sufficient space in the destination
|
||||||
|
* buffer to encrypt the entire input buffer.
|
||||||
|
* <p/>
|
||||||
|
* Upon return, inBuffer.position() will be advanced by the number of bytes
|
||||||
|
* read and outBuffer.position() by bytes written. Implementations should
|
||||||
|
* not modify inBuffer.limit() and outBuffer.limit().
|
||||||
|
* <p/>
|
||||||
|
* @param inBuffer a direct {@link ByteBuffer} to read from. inBuffer may
|
||||||
|
* not be null and inBuffer.remaining() must be > 0
|
||||||
|
* @param outBuffer a direct {@link ByteBuffer} to write to. outBuffer may
|
||||||
|
* not be null and outBuffer.remaining() must be > 0
|
||||||
|
* @throws IOException if encryption fails
|
||||||
|
*/
|
||||||
|
public void encrypt(ByteBuffer inBuffer, ByteBuffer outBuffer)
|
||||||
|
throws IOException;
|
||||||
|
}
|
|
@ -0,0 +1,165 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.crypto;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.ByteBuffer;
|
||||||
|
import java.security.GeneralSecurityException;
|
||||||
|
import java.security.SecureRandom;
|
||||||
|
|
||||||
|
import javax.crypto.Cipher;
|
||||||
|
import javax.crypto.spec.IvParameterSpec;
|
||||||
|
import javax.crypto.spec.SecretKeySpec;
|
||||||
|
|
||||||
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
|
||||||
|
import com.google.common.base.Preconditions;
|
||||||
|
|
||||||
|
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY;
|
||||||
|
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY;
|
||||||
|
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_DEFAULT;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Implement the AES-CTR crypto codec using JCE provider.
|
||||||
|
*/
|
||||||
|
@InterfaceAudience.Private
|
||||||
|
public class JceAesCtrCryptoCodec extends AesCtrCryptoCodec {
|
||||||
|
private static final Log LOG =
|
||||||
|
LogFactory.getLog(JceAesCtrCryptoCodec.class.getName());
|
||||||
|
|
||||||
|
private Configuration conf;
|
||||||
|
private String provider;
|
||||||
|
private SecureRandom random;
|
||||||
|
|
||||||
|
public JceAesCtrCryptoCodec() {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Configuration getConf() {
|
||||||
|
return conf;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setConf(Configuration conf) {
|
||||||
|
this.conf = conf;
|
||||||
|
provider = conf.get(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY);
|
||||||
|
final String secureRandomAlg = conf.get(
|
||||||
|
HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY,
|
||||||
|
HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_DEFAULT);
|
||||||
|
try {
|
||||||
|
random = (provider != null) ?
|
||||||
|
SecureRandom.getInstance(secureRandomAlg, provider) :
|
||||||
|
SecureRandom.getInstance(secureRandomAlg);
|
||||||
|
} catch (GeneralSecurityException e) {
|
||||||
|
LOG.warn(e.getMessage());
|
||||||
|
random = new SecureRandom();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Encryptor createEncryptor() throws GeneralSecurityException {
|
||||||
|
return new JceAesCtrCipher(Cipher.ENCRYPT_MODE, provider);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Decryptor createDecryptor() throws GeneralSecurityException {
|
||||||
|
return new JceAesCtrCipher(Cipher.DECRYPT_MODE, provider);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void generateSecureRandom(byte[] bytes) {
|
||||||
|
random.nextBytes(bytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static class JceAesCtrCipher implements Encryptor, Decryptor {
|
||||||
|
private final Cipher cipher;
|
||||||
|
private final int mode;
|
||||||
|
private boolean contextReset = false;
|
||||||
|
|
||||||
|
public JceAesCtrCipher(int mode, String provider)
|
||||||
|
throws GeneralSecurityException {
|
||||||
|
this.mode = mode;
|
||||||
|
if (provider == null || provider.isEmpty()) {
|
||||||
|
cipher = Cipher.getInstance(SUITE.getName());
|
||||||
|
} else {
|
||||||
|
cipher = Cipher.getInstance(SUITE.getName(), provider);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init(byte[] key, byte[] iv) throws IOException {
|
||||||
|
Preconditions.checkNotNull(key);
|
||||||
|
Preconditions.checkNotNull(iv);
|
||||||
|
contextReset = false;
|
||||||
|
try {
|
||||||
|
cipher.init(mode, new SecretKeySpec(key, "AES"),
|
||||||
|
new IvParameterSpec(iv));
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new IOException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* AES-CTR will consume all of the input data. It requires enough space in
|
||||||
|
* the destination buffer to encrypt entire input buffer.
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public void encrypt(ByteBuffer inBuffer, ByteBuffer outBuffer)
|
||||||
|
throws IOException {
|
||||||
|
process(inBuffer, outBuffer);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* AES-CTR will consume all of the input data. It requires enough space in
|
||||||
|
* the destination buffer to decrypt entire input buffer.
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public void decrypt(ByteBuffer inBuffer, ByteBuffer outBuffer)
|
||||||
|
throws IOException {
|
||||||
|
process(inBuffer, outBuffer);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void process(ByteBuffer inBuffer, ByteBuffer outBuffer)
|
||||||
|
throws IOException {
|
||||||
|
try {
|
||||||
|
int inputSize = inBuffer.remaining();
|
||||||
|
// Cipher#update will maintain crypto context.
|
||||||
|
int n = cipher.update(inBuffer, outBuffer);
|
||||||
|
if (n < inputSize) {
|
||||||
|
/**
|
||||||
|
* Typically code will not get here. Cipher#update will consume all
|
||||||
|
* input data and put result in outBuffer.
|
||||||
|
* Cipher#doFinal will reset the crypto context.
|
||||||
|
*/
|
||||||
|
contextReset = true;
|
||||||
|
cipher.doFinal(inBuffer, outBuffer);
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new IOException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean isContextReset() {
|
||||||
|
return contextReset;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,164 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.crypto;
|
||||||
|
|
||||||
|
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_SECURE_RANDOM_IMPL_KEY;
|
||||||
|
|
||||||
|
import java.io.Closeable;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.ByteBuffer;
|
||||||
|
import java.security.GeneralSecurityException;
|
||||||
|
import java.security.SecureRandom;
|
||||||
|
import java.util.Random;
|
||||||
|
|
||||||
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
|
||||||
|
import com.google.common.base.Preconditions;
|
||||||
|
import org.apache.hadoop.crypto.random.OsSecureRandom;
|
||||||
|
import org.apache.hadoop.util.ReflectionUtils;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Implement the AES-CTR crypto codec using JNI into OpenSSL.
|
||||||
|
*/
|
||||||
|
@InterfaceAudience.Private
|
||||||
|
public class OpensslAesCtrCryptoCodec extends AesCtrCryptoCodec {
|
||||||
|
private static final Log LOG =
|
||||||
|
LogFactory.getLog(OpensslAesCtrCryptoCodec.class.getName());
|
||||||
|
|
||||||
|
private Configuration conf;
|
||||||
|
private Random random;
|
||||||
|
|
||||||
|
public OpensslAesCtrCryptoCodec() {
|
||||||
|
String loadingFailureReason = OpensslCipher.getLoadingFailureReason();
|
||||||
|
if (loadingFailureReason != null) {
|
||||||
|
throw new RuntimeException(loadingFailureReason);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setConf(Configuration conf) {
|
||||||
|
this.conf = conf;
|
||||||
|
final Class<? extends Random> klass = conf.getClass(
|
||||||
|
HADOOP_SECURITY_SECURE_RANDOM_IMPL_KEY, OsSecureRandom.class,
|
||||||
|
Random.class);
|
||||||
|
try {
|
||||||
|
random = ReflectionUtils.newInstance(klass, conf);
|
||||||
|
} catch (Exception e) {
|
||||||
|
LOG.info("Unable to use " + klass.getName() + ". Falling back to " +
|
||||||
|
"Java SecureRandom.", e);
|
||||||
|
this.random = new SecureRandom();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void finalize() throws Throwable {
|
||||||
|
try {
|
||||||
|
Closeable r = (Closeable) this.random;
|
||||||
|
r.close();
|
||||||
|
} catch (ClassCastException e) {
|
||||||
|
}
|
||||||
|
super.finalize();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Configuration getConf() {
|
||||||
|
return conf;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Encryptor createEncryptor() throws GeneralSecurityException {
|
||||||
|
return new OpensslAesCtrCipher(OpensslCipher.ENCRYPT_MODE);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Decryptor createDecryptor() throws GeneralSecurityException {
|
||||||
|
return new OpensslAesCtrCipher(OpensslCipher.DECRYPT_MODE);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void generateSecureRandom(byte[] bytes) {
|
||||||
|
random.nextBytes(bytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static class OpensslAesCtrCipher implements Encryptor, Decryptor {
|
||||||
|
private final OpensslCipher cipher;
|
||||||
|
private final int mode;
|
||||||
|
private boolean contextReset = false;
|
||||||
|
|
||||||
|
public OpensslAesCtrCipher(int mode) throws GeneralSecurityException {
|
||||||
|
this.mode = mode;
|
||||||
|
cipher = OpensslCipher.getInstance(SUITE.getName());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init(byte[] key, byte[] iv) throws IOException {
|
||||||
|
Preconditions.checkNotNull(key);
|
||||||
|
Preconditions.checkNotNull(iv);
|
||||||
|
contextReset = false;
|
||||||
|
cipher.init(mode, key, iv);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* AES-CTR will consume all of the input data. It requires enough space in
|
||||||
|
* the destination buffer to encrypt entire input buffer.
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public void encrypt(ByteBuffer inBuffer, ByteBuffer outBuffer)
|
||||||
|
throws IOException {
|
||||||
|
process(inBuffer, outBuffer);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* AES-CTR will consume all of the input data. It requires enough space in
|
||||||
|
* the destination buffer to decrypt entire input buffer.
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public void decrypt(ByteBuffer inBuffer, ByteBuffer outBuffer)
|
||||||
|
throws IOException {
|
||||||
|
process(inBuffer, outBuffer);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void process(ByteBuffer inBuffer, ByteBuffer outBuffer)
|
||||||
|
throws IOException {
|
||||||
|
try {
|
||||||
|
int inputSize = inBuffer.remaining();
|
||||||
|
// OpensslCipher#update will maintain crypto context.
|
||||||
|
int n = cipher.update(inBuffer, outBuffer);
|
||||||
|
if (n < inputSize) {
|
||||||
|
/**
|
||||||
|
* Typically code will not get here. OpensslCipher#update will
|
||||||
|
* consume all input data and put result in outBuffer.
|
||||||
|
* OpensslCipher#doFinal will reset the crypto context.
|
||||||
|
*/
|
||||||
|
contextReset = true;
|
||||||
|
cipher.doFinal(outBuffer);
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new IOException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean isContextReset() {
|
||||||
|
return contextReset;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,287 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.crypto;
|
||||||
|
|
||||||
|
import java.nio.ByteBuffer;
|
||||||
|
import java.security.NoSuchAlgorithmException;
|
||||||
|
import java.util.StringTokenizer;
|
||||||
|
|
||||||
|
import javax.crypto.BadPaddingException;
|
||||||
|
import javax.crypto.IllegalBlockSizeException;
|
||||||
|
import javax.crypto.NoSuchPaddingException;
|
||||||
|
import javax.crypto.ShortBufferException;
|
||||||
|
|
||||||
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
|
import org.apache.hadoop.util.NativeCodeLoader;
|
||||||
|
|
||||||
|
import com.google.common.base.Preconditions;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* OpenSSL cipher using JNI.
|
||||||
|
* Currently only AES-CTR is supported. It's flexible to add
|
||||||
|
* other crypto algorithms/modes.
|
||||||
|
*/
|
||||||
|
@InterfaceAudience.Private
|
||||||
|
public final class OpensslCipher {
|
||||||
|
private static final Log LOG =
|
||||||
|
LogFactory.getLog(OpensslCipher.class.getName());
|
||||||
|
public static final int ENCRYPT_MODE = 1;
|
||||||
|
public static final int DECRYPT_MODE = 0;
|
||||||
|
|
||||||
|
/** Currently only support AES/CTR/NoPadding. */
|
||||||
|
private static enum AlgMode {
|
||||||
|
AES_CTR;
|
||||||
|
|
||||||
|
static int get(String algorithm, String mode)
|
||||||
|
throws NoSuchAlgorithmException {
|
||||||
|
try {
|
||||||
|
return AlgMode.valueOf(algorithm + "_" + mode).ordinal();
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new NoSuchAlgorithmException("Doesn't support algorithm: " +
|
||||||
|
algorithm + " and mode: " + mode);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static enum Padding {
|
||||||
|
NoPadding;
|
||||||
|
|
||||||
|
static int get(String padding) throws NoSuchPaddingException {
|
||||||
|
try {
|
||||||
|
return Padding.valueOf(padding).ordinal();
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new NoSuchPaddingException("Doesn't support padding: " + padding);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private long context = 0;
|
||||||
|
private final int alg;
|
||||||
|
private final int padding;
|
||||||
|
|
||||||
|
private static final String loadingFailureReason;
|
||||||
|
|
||||||
|
static {
|
||||||
|
String loadingFailure = null;
|
||||||
|
try {
|
||||||
|
if (!NativeCodeLoader.buildSupportsOpenssl()) {
|
||||||
|
loadingFailure = "build does not support openssl.";
|
||||||
|
} else {
|
||||||
|
initIDs();
|
||||||
|
}
|
||||||
|
} catch (Throwable t) {
|
||||||
|
loadingFailure = t.getMessage();
|
||||||
|
LOG.debug("Failed to load OpenSSL Cipher.", t);
|
||||||
|
} finally {
|
||||||
|
loadingFailureReason = loadingFailure;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getLoadingFailureReason() {
|
||||||
|
return loadingFailureReason;
|
||||||
|
}
|
||||||
|
|
||||||
|
private OpensslCipher(long context, int alg, int padding) {
|
||||||
|
this.context = context;
|
||||||
|
this.alg = alg;
|
||||||
|
this.padding = padding;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return an <code>OpensslCipher<code> object that implements the specified
|
||||||
|
* transformation.
|
||||||
|
*
|
||||||
|
* @param transformation the name of the transformation, e.g.,
|
||||||
|
* AES/CTR/NoPadding.
|
||||||
|
* @return OpensslCipher an <code>OpensslCipher<code> object
|
||||||
|
* @throws NoSuchAlgorithmException if <code>transformation</code> is null,
|
||||||
|
* empty, in an invalid format, or if Openssl doesn't implement the
|
||||||
|
* specified algorithm.
|
||||||
|
* @throws NoSuchPaddingException if <code>transformation</code> contains
|
||||||
|
* a padding scheme that is not available.
|
||||||
|
*/
|
||||||
|
public static final OpensslCipher getInstance(String transformation)
|
||||||
|
throws NoSuchAlgorithmException, NoSuchPaddingException {
|
||||||
|
Transform transform = tokenizeTransformation(transformation);
|
||||||
|
int algMode = AlgMode.get(transform.alg, transform.mode);
|
||||||
|
int padding = Padding.get(transform.padding);
|
||||||
|
long context = initContext(algMode, padding);
|
||||||
|
return new OpensslCipher(context, algMode, padding);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Nested class for algorithm, mode and padding. */
|
||||||
|
private static class Transform {
|
||||||
|
final String alg;
|
||||||
|
final String mode;
|
||||||
|
final String padding;
|
||||||
|
|
||||||
|
public Transform(String alg, String mode, String padding) {
|
||||||
|
this.alg = alg;
|
||||||
|
this.mode = mode;
|
||||||
|
this.padding = padding;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Transform tokenizeTransformation(String transformation)
|
||||||
|
throws NoSuchAlgorithmException {
|
||||||
|
if (transformation == null) {
|
||||||
|
throw new NoSuchAlgorithmException("No transformation given.");
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Array containing the components of a Cipher transformation:
|
||||||
|
*
|
||||||
|
* index 0: algorithm (e.g., AES)
|
||||||
|
* index 1: mode (e.g., CTR)
|
||||||
|
* index 2: padding (e.g., NoPadding)
|
||||||
|
*/
|
||||||
|
String[] parts = new String[3];
|
||||||
|
int count = 0;
|
||||||
|
StringTokenizer parser = new StringTokenizer(transformation, "/");
|
||||||
|
while (parser.hasMoreTokens() && count < 3) {
|
||||||
|
parts[count++] = parser.nextToken().trim();
|
||||||
|
}
|
||||||
|
if (count != 3 || parser.hasMoreTokens()) {
|
||||||
|
throw new NoSuchAlgorithmException("Invalid transformation format: " +
|
||||||
|
transformation);
|
||||||
|
}
|
||||||
|
return new Transform(parts[0], parts[1], parts[2]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Initialize this cipher with a key and IV.
|
||||||
|
*
|
||||||
|
* @param mode {@link #ENCRYPT_MODE} or {@link #DECRYPT_MODE}
|
||||||
|
* @param key crypto key
|
||||||
|
* @param iv crypto iv
|
||||||
|
*/
|
||||||
|
public void init(int mode, byte[] key, byte[] iv) {
|
||||||
|
context = init(context, mode, alg, padding, key, iv);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Continues a multiple-part encryption or decryption operation. The data
|
||||||
|
* is encrypted or decrypted, depending on how this cipher was initialized.
|
||||||
|
* <p/>
|
||||||
|
*
|
||||||
|
* All <code>input.remaining()</code> bytes starting at
|
||||||
|
* <code>input.position()</code> are processed. The result is stored in
|
||||||
|
* the output buffer.
|
||||||
|
* <p/>
|
||||||
|
*
|
||||||
|
* Upon return, the input buffer's position will be equal to its limit;
|
||||||
|
* its limit will not have changed. The output buffer's position will have
|
||||||
|
* advanced by n, when n is the value returned by this method; the output
|
||||||
|
* buffer's limit will not have changed.
|
||||||
|
* <p/>
|
||||||
|
*
|
||||||
|
* If <code>output.remaining()</code> bytes are insufficient to hold the
|
||||||
|
* result, a <code>ShortBufferException</code> is thrown.
|
||||||
|
*
|
||||||
|
* @param input the input ByteBuffer
|
||||||
|
* @param output the output ByteBuffer
|
||||||
|
* @return int number of bytes stored in <code>output</code>
|
||||||
|
* @throws ShortBufferException if there is insufficient space in the
|
||||||
|
* output buffer
|
||||||
|
*/
|
||||||
|
public int update(ByteBuffer input, ByteBuffer output)
|
||||||
|
throws ShortBufferException {
|
||||||
|
checkState();
|
||||||
|
Preconditions.checkArgument(input.isDirect() && output.isDirect(),
|
||||||
|
"Direct buffers are required.");
|
||||||
|
int len = update(context, input, input.position(), input.remaining(),
|
||||||
|
output, output.position(), output.remaining());
|
||||||
|
input.position(input.limit());
|
||||||
|
output.position(output.position() + len);
|
||||||
|
return len;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Finishes a multiple-part operation. The data is encrypted or decrypted,
|
||||||
|
* depending on how this cipher was initialized.
|
||||||
|
* <p/>
|
||||||
|
*
|
||||||
|
* The result is stored in the output buffer. Upon return, the output buffer's
|
||||||
|
* position will have advanced by n, where n is the value returned by this
|
||||||
|
* method; the output buffer's limit will not have changed.
|
||||||
|
* <p/>
|
||||||
|
*
|
||||||
|
* If <code>output.remaining()</code> bytes are insufficient to hold the result,
|
||||||
|
* a <code>ShortBufferException</code> is thrown.
|
||||||
|
* <p/>
|
||||||
|
*
|
||||||
|
* Upon finishing, this method resets this cipher object to the state it was
|
||||||
|
* in when previously initialized. That is, the object is available to encrypt
|
||||||
|
* or decrypt more data.
|
||||||
|
* <p/>
|
||||||
|
*
|
||||||
|
* If any exception is thrown, this cipher object need to be reset before it
|
||||||
|
* can be used again.
|
||||||
|
*
|
||||||
|
* @param output the output ByteBuffer
|
||||||
|
* @return int number of bytes stored in <code>output</code>
|
||||||
|
* @throws ShortBufferException
|
||||||
|
* @throws IllegalBlockSizeException
|
||||||
|
* @throws BadPaddingException
|
||||||
|
*/
|
||||||
|
public int doFinal(ByteBuffer output) throws ShortBufferException,
|
||||||
|
IllegalBlockSizeException, BadPaddingException {
|
||||||
|
checkState();
|
||||||
|
Preconditions.checkArgument(output.isDirect(), "Direct buffer is required.");
|
||||||
|
int len = doFinal(context, output, output.position(), output.remaining());
|
||||||
|
output.position(output.position() + len);
|
||||||
|
return len;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Forcibly clean the context. */
|
||||||
|
public void clean() {
|
||||||
|
if (context != 0) {
|
||||||
|
clean(context);
|
||||||
|
context = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Check whether context is initialized. */
|
||||||
|
private void checkState() {
|
||||||
|
Preconditions.checkState(context != 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void finalize() throws Throwable {
|
||||||
|
clean();
|
||||||
|
}
|
||||||
|
|
||||||
|
private native static void initIDs();
|
||||||
|
|
||||||
|
private native static long initContext(int alg, int padding);
|
||||||
|
|
||||||
|
private native long init(long context, int mode, int alg, int padding,
|
||||||
|
byte[] key, byte[] iv);
|
||||||
|
|
||||||
|
private native int update(long context, ByteBuffer input, int inputOffset,
|
||||||
|
int inputLength, ByteBuffer output, int outputOffset, int maxOutputLength);
|
||||||
|
|
||||||
|
private native int doFinal(long context, ByteBuffer output, int offset,
|
||||||
|
int maxOutputLength);
|
||||||
|
|
||||||
|
private native void clean(long context);
|
||||||
|
|
||||||
|
public native static String getLibraryName();
|
||||||
|
}
|
|
@ -0,0 +1,119 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.crypto.random;
|
||||||
|
|
||||||
|
import java.util.Random;
|
||||||
|
|
||||||
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
|
import org.apache.hadoop.util.NativeCodeLoader;
|
||||||
|
|
||||||
|
import com.google.common.base.Preconditions;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* OpenSSL secure random using JNI.
|
||||||
|
* This implementation is thread-safe.
|
||||||
|
* <p/>
|
||||||
|
*
|
||||||
|
* If using an Intel chipset with RDRAND, the high-performance hardware
|
||||||
|
* random number generator will be used and it's much faster than
|
||||||
|
* {@link java.security.SecureRandom}. If RDRAND is unavailable, default
|
||||||
|
* OpenSSL secure random generator will be used. It's still faster
|
||||||
|
* and can generate strong random bytes.
|
||||||
|
* <p/>
|
||||||
|
* @see https://wiki.openssl.org/index.php/Random_Numbers
|
||||||
|
* @see http://en.wikipedia.org/wiki/RdRand
|
||||||
|
*/
|
||||||
|
@InterfaceAudience.Private
|
||||||
|
public class OpensslSecureRandom extends Random {
|
||||||
|
private static final long serialVersionUID = -7828193502768789584L;
|
||||||
|
private static final Log LOG =
|
||||||
|
LogFactory.getLog(OpensslSecureRandom.class.getName());
|
||||||
|
|
||||||
|
/** If native SecureRandom unavailable, use java SecureRandom */
|
||||||
|
private java.security.SecureRandom fallback = null;
|
||||||
|
private static boolean nativeEnabled = false;
|
||||||
|
static {
|
||||||
|
if (NativeCodeLoader.isNativeCodeLoaded() &&
|
||||||
|
NativeCodeLoader.buildSupportsOpenssl()) {
|
||||||
|
try {
|
||||||
|
initSR();
|
||||||
|
nativeEnabled = true;
|
||||||
|
} catch (Throwable t) {
|
||||||
|
LOG.error("Failed to load Openssl SecureRandom", t);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static boolean isNativeCodeLoaded() {
|
||||||
|
return nativeEnabled;
|
||||||
|
}
|
||||||
|
|
||||||
|
public OpensslSecureRandom() {
|
||||||
|
if (!nativeEnabled) {
|
||||||
|
fallback = new java.security.SecureRandom();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generates a user-specified number of random bytes.
|
||||||
|
* It's thread-safe.
|
||||||
|
*
|
||||||
|
* @param bytes the array to be filled in with random bytes.
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public void nextBytes(byte[] bytes) {
|
||||||
|
if (!nativeEnabled || !nextRandBytes(bytes)) {
|
||||||
|
fallback.nextBytes(bytes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setSeed(long seed) {
|
||||||
|
// Self-seeding.
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generates an integer containing the user-specified number of
|
||||||
|
* random bits (right justified, with leading zeros).
|
||||||
|
*
|
||||||
|
* @param numBits number of random bits to be generated, where
|
||||||
|
* 0 <= <code>numBits</code> <= 32.
|
||||||
|
*
|
||||||
|
* @return int an <code>int</code> containing the user-specified number
|
||||||
|
* of random bits (right justified, with leading zeros).
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
final protected int next(int numBits) {
|
||||||
|
Preconditions.checkArgument(numBits >= 0 && numBits <= 32);
|
||||||
|
int numBytes = (numBits + 7) / 8;
|
||||||
|
byte b[] = new byte[numBytes];
|
||||||
|
int next = 0;
|
||||||
|
|
||||||
|
nextBytes(b);
|
||||||
|
for (int i = 0; i < numBytes; i++) {
|
||||||
|
next = (next << 8) + (b[i] & 0xFF);
|
||||||
|
}
|
||||||
|
|
||||||
|
return next >>> (numBytes * 8 - numBits);
|
||||||
|
}
|
||||||
|
|
||||||
|
private native static void initSR();
|
||||||
|
private native boolean nextRandBytes(byte[] bytes);
|
||||||
|
}
|
|
@ -0,0 +1,115 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.crypto.random;
|
||||||
|
|
||||||
|
import java.io.Closeable;
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileInputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Random;
|
||||||
|
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
|
import org.apache.hadoop.conf.Configurable;
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.io.IOUtils;
|
||||||
|
|
||||||
|
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_KEY;
|
||||||
|
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_DEFAULT;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A Random implementation that uses random bytes sourced from the
|
||||||
|
* operating system.
|
||||||
|
*/
|
||||||
|
@InterfaceAudience.Private
|
||||||
|
public class OsSecureRandom extends Random implements Closeable, Configurable {
|
||||||
|
private static final long serialVersionUID = 6391500337172057900L;
|
||||||
|
|
||||||
|
private transient Configuration conf;
|
||||||
|
|
||||||
|
private final int RESERVOIR_LENGTH = 8192;
|
||||||
|
|
||||||
|
private String randomDevPath;
|
||||||
|
|
||||||
|
private transient FileInputStream stream;
|
||||||
|
|
||||||
|
private final byte[] reservoir = new byte[RESERVOIR_LENGTH];
|
||||||
|
|
||||||
|
private int pos = reservoir.length;
|
||||||
|
|
||||||
|
private void fillReservoir(int min) {
|
||||||
|
if (pos >= reservoir.length - min) {
|
||||||
|
try {
|
||||||
|
IOUtils.readFully(stream, reservoir, 0, reservoir.length);
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new RuntimeException("failed to fill reservoir", e);
|
||||||
|
}
|
||||||
|
pos = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public OsSecureRandom() {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
synchronized public void setConf(Configuration conf) {
|
||||||
|
this.conf = conf;
|
||||||
|
this.randomDevPath = conf.get(
|
||||||
|
HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_KEY,
|
||||||
|
HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_DEFAULT);
|
||||||
|
File randomDevFile = new File(randomDevPath);
|
||||||
|
try {
|
||||||
|
this.stream = new FileInputStream(randomDevFile);
|
||||||
|
fillReservoir(0);
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
synchronized public Configuration getConf() {
|
||||||
|
return conf;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
synchronized public void nextBytes(byte[] bytes) {
|
||||||
|
int off = 0;
|
||||||
|
int n = 0;
|
||||||
|
while (off < bytes.length) {
|
||||||
|
fillReservoir(0);
|
||||||
|
n = Math.min(bytes.length - off, reservoir.length - pos);
|
||||||
|
System.arraycopy(reservoir, pos, bytes, off, n);
|
||||||
|
off += n;
|
||||||
|
pos += n;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
synchronized protected int next(int nbits) {
|
||||||
|
fillReservoir(4);
|
||||||
|
int n = 0;
|
||||||
|
for (int i = 0; i < 4; i++) {
|
||||||
|
n = ((n << 8) | (reservoir[pos++] & 0xff));
|
||||||
|
}
|
||||||
|
return n & (0xffffffff >> (32 - nbits));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
synchronized public void close() throws IOException {
|
||||||
|
stream.close();
|
||||||
|
}
|
||||||
|
}
|
|
@ -283,5 +283,4 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
|
||||||
public static final String NFS_EXPORTS_ALLOWED_HOSTS_SEPARATOR = ";";
|
public static final String NFS_EXPORTS_ALLOWED_HOSTS_SEPARATOR = ";";
|
||||||
public static final String NFS_EXPORTS_ALLOWED_HOSTS_KEY = "nfs.exports.allowed.hosts";
|
public static final String NFS_EXPORTS_ALLOWED_HOSTS_KEY = "nfs.exports.allowed.hosts";
|
||||||
public static final String NFS_EXPORTS_ALLOWED_HOSTS_KEY_DEFAULT = "* rw";
|
public static final String NFS_EXPORTS_ALLOWED_HOSTS_KEY_DEFAULT = "* rw";
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -288,6 +288,21 @@ public class CommonConfigurationKeysPublic {
|
||||||
/** Class to override Sasl Properties for a connection */
|
/** Class to override Sasl Properties for a connection */
|
||||||
public static final String HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS =
|
public static final String HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS =
|
||||||
"hadoop.security.saslproperties.resolver.class";
|
"hadoop.security.saslproperties.resolver.class";
|
||||||
|
public static final String HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX =
|
||||||
|
"hadoop.security.crypto.codec.classes";
|
||||||
|
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
|
||||||
|
public static final String HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY =
|
||||||
|
"hadoop.security.crypto.cipher.suite";
|
||||||
|
public static final String HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_DEFAULT =
|
||||||
|
"AES/CTR/NoPadding";
|
||||||
|
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
|
||||||
|
public static final String HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY =
|
||||||
|
"hadoop.security.crypto.jce.provider";
|
||||||
|
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
|
||||||
|
public static final String HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_KEY =
|
||||||
|
"hadoop.security.crypto.buffer.size";
|
||||||
|
/** Defalt value for HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_KEY */
|
||||||
|
public static final int HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_DEFAULT = 8192;
|
||||||
/** Class to override Impersonation provider */
|
/** Class to override Impersonation provider */
|
||||||
public static final String HADOOP_SECURITY_IMPERSONATION_PROVIDER_CLASS =
|
public static final String HADOOP_SECURITY_IMPERSONATION_PROVIDER_CLASS =
|
||||||
"hadoop.security.impersonation.provider.class";
|
"hadoop.security.impersonation.provider.class";
|
||||||
|
@ -318,5 +333,20 @@ public class CommonConfigurationKeysPublic {
|
||||||
"hadoop.security.kms.client.encrypted.key.cache.expiry";
|
"hadoop.security.kms.client.encrypted.key.cache.expiry";
|
||||||
/** Default value for KMS_CLIENT_ENC_KEY_CACHE_EXPIRY (12 hrs)*/
|
/** Default value for KMS_CLIENT_ENC_KEY_CACHE_EXPIRY (12 hrs)*/
|
||||||
public static final int KMS_CLIENT_ENC_KEY_CACHE_EXPIRY_DEFAULT = 43200000;
|
public static final int KMS_CLIENT_ENC_KEY_CACHE_EXPIRY_DEFAULT = 43200000;
|
||||||
|
|
||||||
|
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
|
||||||
|
public static final String HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY =
|
||||||
|
"hadoop.security.java.secure.random.algorithm";
|
||||||
|
/** Defalt value for HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY */
|
||||||
|
public static final String HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_DEFAULT =
|
||||||
|
"SHA1PRNG";
|
||||||
|
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
|
||||||
|
public static final String HADOOP_SECURITY_SECURE_RANDOM_IMPL_KEY =
|
||||||
|
"hadoop.security.secure.random.impl";
|
||||||
|
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
|
||||||
|
public static final String HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_KEY =
|
||||||
|
"hadoop.security.random.device.file.path";
|
||||||
|
public static final String HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_DEFAULT =
|
||||||
|
"/dev/urandom";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -102,7 +102,7 @@ public class FSDataOutputStream extends DataOutputStream
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get a reference to the wrapped output stream. Used by unit tests.
|
* Get a reference to the wrapped output stream.
|
||||||
*
|
*
|
||||||
* @return the underlying output stream
|
* @return the underlying output stream
|
||||||
*/
|
*/
|
||||||
|
|
|
@ -0,0 +1,102 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.fs;
|
||||||
|
|
||||||
|
import org.apache.commons.codec.binary.Hex;
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
|
import org.apache.hadoop.crypto.CipherSuite;
|
||||||
|
|
||||||
|
import static com.google.common.base.Preconditions.checkArgument;
|
||||||
|
import static com.google.common.base.Preconditions.checkNotNull;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* FileEncryptionInfo encapsulates all the encryption-related information for
|
||||||
|
* an encrypted file.
|
||||||
|
*/
|
||||||
|
@InterfaceAudience.Private
|
||||||
|
public class FileEncryptionInfo {
|
||||||
|
|
||||||
|
private final CipherSuite cipherSuite;
|
||||||
|
private final byte[] edek;
|
||||||
|
private final byte[] iv;
|
||||||
|
private final String ezKeyVersionName;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a FileEncryptionInfo.
|
||||||
|
*
|
||||||
|
* @param suite CipherSuite used to encrypt the file
|
||||||
|
* @param edek encrypted data encryption key (EDEK) of the file
|
||||||
|
* @param iv initialization vector (IV) used to encrypt the file
|
||||||
|
* @param ezKeyVersionName name of the KeyVersion used to encrypt the
|
||||||
|
* encrypted data encryption key.
|
||||||
|
*/
|
||||||
|
public FileEncryptionInfo(final CipherSuite suite, final byte[] edek,
|
||||||
|
final byte[] iv, final String ezKeyVersionName) {
|
||||||
|
checkNotNull(suite);
|
||||||
|
checkNotNull(edek);
|
||||||
|
checkNotNull(iv);
|
||||||
|
checkNotNull(ezKeyVersionName);
|
||||||
|
checkArgument(edek.length == suite.getAlgorithmBlockSize(),
|
||||||
|
"Unexpected key length");
|
||||||
|
checkArgument(iv.length == suite.getAlgorithmBlockSize(),
|
||||||
|
"Unexpected IV length");
|
||||||
|
this.cipherSuite = suite;
|
||||||
|
this.edek = edek;
|
||||||
|
this.iv = iv;
|
||||||
|
this.ezKeyVersionName = ezKeyVersionName;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return {@link org.apache.hadoop.crypto.CipherSuite} used to encrypt
|
||||||
|
* the file.
|
||||||
|
*/
|
||||||
|
public CipherSuite getCipherSuite() {
|
||||||
|
return cipherSuite;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return encrypted data encryption key (EDEK) for the file
|
||||||
|
*/
|
||||||
|
public byte[] getEncryptedDataEncryptionKey() {
|
||||||
|
return edek;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return initialization vector (IV) for the cipher used to encrypt the file
|
||||||
|
*/
|
||||||
|
public byte[] getIV() {
|
||||||
|
return iv;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return name of the encryption zone KeyVersion used to encrypt the
|
||||||
|
* encrypted data encryption key (EDEK).
|
||||||
|
*/
|
||||||
|
public String getEzKeyVersionName() { return ezKeyVersionName; }
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
StringBuilder builder = new StringBuilder("{");
|
||||||
|
builder.append("cipherSuite: " + cipherSuite);
|
||||||
|
builder.append(", edek: " + Hex.encodeHexString(edek));
|
||||||
|
builder.append(", iv: " + Hex.encodeHexString(iv));
|
||||||
|
builder.append(", ezKeyVersionName: " + ezKeyVersionName);
|
||||||
|
builder.append("}");
|
||||||
|
return builder.toString();
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,37 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.fs.crypto;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import org.apache.hadoop.crypto.CryptoCodec;
|
||||||
|
import org.apache.hadoop.crypto.CryptoInputStream;
|
||||||
|
import org.apache.hadoop.fs.FSDataInputStream;
|
||||||
|
|
||||||
|
public class CryptoFSDataInputStream extends FSDataInputStream {
|
||||||
|
|
||||||
|
public CryptoFSDataInputStream(FSDataInputStream in, CryptoCodec codec,
|
||||||
|
int bufferSize, byte[] key, byte[] iv) throws IOException {
|
||||||
|
super(new CryptoInputStream(in, codec, bufferSize, key, iv));
|
||||||
|
}
|
||||||
|
|
||||||
|
public CryptoFSDataInputStream(FSDataInputStream in, CryptoCodec codec,
|
||||||
|
byte[] key, byte[] iv) throws IOException {
|
||||||
|
super(new CryptoInputStream(in, codec, key, iv));
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,47 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.fs.crypto;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import org.apache.hadoop.crypto.CryptoCodec;
|
||||||
|
import org.apache.hadoop.crypto.CryptoOutputStream;
|
||||||
|
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||||
|
|
||||||
|
public class CryptoFSDataOutputStream extends FSDataOutputStream {
|
||||||
|
private final FSDataOutputStream fsOut;
|
||||||
|
|
||||||
|
public CryptoFSDataOutputStream(FSDataOutputStream out, CryptoCodec codec,
|
||||||
|
int bufferSize, byte[] key, byte[] iv) throws IOException {
|
||||||
|
super(new CryptoOutputStream(out, codec, bufferSize, key, iv,
|
||||||
|
out.getPos()), null, out.getPos());
|
||||||
|
this.fsOut = out;
|
||||||
|
}
|
||||||
|
|
||||||
|
public CryptoFSDataOutputStream(FSDataOutputStream out, CryptoCodec codec,
|
||||||
|
byte[] key, byte[] iv) throws IOException {
|
||||||
|
super(new CryptoOutputStream(out, codec, key, iv, out.getPos()),
|
||||||
|
null, out.getPos());
|
||||||
|
this.fsOut = out;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public long getPos() {
|
||||||
|
return fsOut.getPos();
|
||||||
|
}
|
||||||
|
}
|
|
@ -57,6 +57,17 @@ abstract class CommandWithDestination extends FsCommand {
|
||||||
private boolean verifyChecksum = true;
|
private boolean verifyChecksum = true;
|
||||||
private boolean writeChecksum = true;
|
private boolean writeChecksum = true;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The name of the raw xattr namespace. It would be nice to use
|
||||||
|
* XAttr.RAW.name() but we can't reference the hadoop-hdfs project.
|
||||||
|
*/
|
||||||
|
private static final String RAW = "raw.";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The name of the reserved raw directory.
|
||||||
|
*/
|
||||||
|
private static final String RESERVED_RAW = "/.reserved/raw";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
*
|
*
|
||||||
* This method is used to enable the force(-f) option while copying the files.
|
* This method is used to enable the force(-f) option while copying the files.
|
||||||
|
@ -231,7 +242,7 @@ abstract class CommandWithDestination extends FsCommand {
|
||||||
/**
|
/**
|
||||||
* Called with a source and target destination pair
|
* Called with a source and target destination pair
|
||||||
* @param src for the operation
|
* @param src for the operation
|
||||||
* @param target for the operation
|
* @param dst for the operation
|
||||||
* @throws IOException if anything goes wrong
|
* @throws IOException if anything goes wrong
|
||||||
*/
|
*/
|
||||||
protected void processPath(PathData src, PathData dst) throws IOException {
|
protected void processPath(PathData src, PathData dst) throws IOException {
|
||||||
|
@ -253,6 +264,8 @@ abstract class CommandWithDestination extends FsCommand {
|
||||||
// modify dst as we descend to append the basename of the
|
// modify dst as we descend to append the basename of the
|
||||||
// current directory being processed
|
// current directory being processed
|
||||||
dst = getTargetPath(src);
|
dst = getTargetPath(src);
|
||||||
|
final boolean preserveRawXattrs =
|
||||||
|
checkPathsForReservedRaw(src.path, dst.path);
|
||||||
if (dst.exists) {
|
if (dst.exists) {
|
||||||
if (!dst.stat.isDirectory()) {
|
if (!dst.stat.isDirectory()) {
|
||||||
throw new PathIsNotDirectoryException(dst.toString());
|
throw new PathIsNotDirectoryException(dst.toString());
|
||||||
|
@ -268,7 +281,7 @@ abstract class CommandWithDestination extends FsCommand {
|
||||||
}
|
}
|
||||||
super.recursePath(src);
|
super.recursePath(src);
|
||||||
if (dst.stat.isDirectory()) {
|
if (dst.stat.isDirectory()) {
|
||||||
preserveAttributes(src, dst);
|
preserveAttributes(src, dst, preserveRawXattrs);
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
dst = savedDst;
|
dst = savedDst;
|
||||||
|
@ -295,18 +308,60 @@ abstract class CommandWithDestination extends FsCommand {
|
||||||
* @param target where to copy the item
|
* @param target where to copy the item
|
||||||
* @throws IOException if copy fails
|
* @throws IOException if copy fails
|
||||||
*/
|
*/
|
||||||
protected void copyFileToTarget(PathData src, PathData target) throws IOException {
|
protected void copyFileToTarget(PathData src, PathData target)
|
||||||
|
throws IOException {
|
||||||
|
final boolean preserveRawXattrs =
|
||||||
|
checkPathsForReservedRaw(src.path, target.path);
|
||||||
src.fs.setVerifyChecksum(verifyChecksum);
|
src.fs.setVerifyChecksum(verifyChecksum);
|
||||||
InputStream in = null;
|
InputStream in = null;
|
||||||
try {
|
try {
|
||||||
in = src.fs.open(src.path);
|
in = src.fs.open(src.path);
|
||||||
copyStreamToTarget(in, target);
|
copyStreamToTarget(in, target);
|
||||||
preserveAttributes(src, target);
|
preserveAttributes(src, target, preserveRawXattrs);
|
||||||
} finally {
|
} finally {
|
||||||
IOUtils.closeStream(in);
|
IOUtils.closeStream(in);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check the source and target paths to ensure that they are either both in
|
||||||
|
* /.reserved/raw or neither in /.reserved/raw. If neither src nor target are
|
||||||
|
* in /.reserved/raw, then return false, indicating not to preserve raw.*
|
||||||
|
* xattrs. If both src/target are in /.reserved/raw, then return true,
|
||||||
|
* indicating raw.* xattrs should be preserved. If only one of src/target is
|
||||||
|
* in /.reserved/raw then throw an exception.
|
||||||
|
*
|
||||||
|
* @param src The source path to check. This should be a fully-qualified
|
||||||
|
* path, not relative.
|
||||||
|
* @param target The target path to check. This should be a fully-qualified
|
||||||
|
* path, not relative.
|
||||||
|
* @return true if raw.* xattrs should be preserved.
|
||||||
|
* @throws PathOperationException is only one of src/target are in
|
||||||
|
* /.reserved/raw.
|
||||||
|
*/
|
||||||
|
private boolean checkPathsForReservedRaw(Path src, Path target)
|
||||||
|
throws PathOperationException {
|
||||||
|
final boolean srcIsRR = Path.getPathWithoutSchemeAndAuthority(src).
|
||||||
|
toString().startsWith(RESERVED_RAW);
|
||||||
|
final boolean dstIsRR = Path.getPathWithoutSchemeAndAuthority(target).
|
||||||
|
toString().startsWith(RESERVED_RAW);
|
||||||
|
boolean preserveRawXattrs = false;
|
||||||
|
if (srcIsRR && !dstIsRR) {
|
||||||
|
final String s = "' copy from '" + RESERVED_RAW + "' to non '" +
|
||||||
|
RESERVED_RAW + "'. Either both source and target must be in '" +
|
||||||
|
RESERVED_RAW + "' or neither.";
|
||||||
|
throw new PathOperationException("'" + src.toString() + s);
|
||||||
|
} else if (!srcIsRR && dstIsRR) {
|
||||||
|
final String s = "' copy from non '" + RESERVED_RAW +"' to '" +
|
||||||
|
RESERVED_RAW + "'. Either both source and target must be in '" +
|
||||||
|
RESERVED_RAW + "' or neither.";
|
||||||
|
throw new PathOperationException("'" + dst.toString() + s);
|
||||||
|
} else if (srcIsRR && dstIsRR) {
|
||||||
|
preserveRawXattrs = true;
|
||||||
|
}
|
||||||
|
return preserveRawXattrs;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Copies the stream contents to a temporary file. If the copy is
|
* Copies the stream contents to a temporary file. If the copy is
|
||||||
* successful, the temporary file will be renamed to the real path,
|
* successful, the temporary file will be renamed to the real path,
|
||||||
|
@ -337,9 +392,11 @@ abstract class CommandWithDestination extends FsCommand {
|
||||||
* attribute to preserve.
|
* attribute to preserve.
|
||||||
* @param src source to preserve
|
* @param src source to preserve
|
||||||
* @param target where to preserve attributes
|
* @param target where to preserve attributes
|
||||||
|
* @param preserveRawXAttrs true if raw.* xattrs should be preserved
|
||||||
* @throws IOException if fails to preserve attributes
|
* @throws IOException if fails to preserve attributes
|
||||||
*/
|
*/
|
||||||
protected void preserveAttributes(PathData src, PathData target)
|
protected void preserveAttributes(PathData src, PathData target,
|
||||||
|
boolean preserveRawXAttrs)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
if (shouldPreserve(FileAttribute.TIMESTAMPS)) {
|
if (shouldPreserve(FileAttribute.TIMESTAMPS)) {
|
||||||
target.fs.setTimes(
|
target.fs.setTimes(
|
||||||
|
@ -369,13 +426,17 @@ abstract class CommandWithDestination extends FsCommand {
|
||||||
target.fs.setAcl(target.path, srcFullEntries);
|
target.fs.setAcl(target.path, srcFullEntries);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (shouldPreserve(FileAttribute.XATTR)) {
|
final boolean preserveXAttrs = shouldPreserve(FileAttribute.XATTR);
|
||||||
|
if (preserveXAttrs || preserveRawXAttrs) {
|
||||||
Map<String, byte[]> srcXAttrs = src.fs.getXAttrs(src.path);
|
Map<String, byte[]> srcXAttrs = src.fs.getXAttrs(src.path);
|
||||||
if (srcXAttrs != null) {
|
if (srcXAttrs != null) {
|
||||||
Iterator<Entry<String, byte[]>> iter = srcXAttrs.entrySet().iterator();
|
Iterator<Entry<String, byte[]>> iter = srcXAttrs.entrySet().iterator();
|
||||||
while (iter.hasNext()) {
|
while (iter.hasNext()) {
|
||||||
Entry<String, byte[]> entry = iter.next();
|
Entry<String, byte[]> entry = iter.next();
|
||||||
target.fs.setXAttr(target.path, entry.getKey(), entry.getValue());
|
final String xattrName = entry.getKey();
|
||||||
|
if (xattrName.startsWith(RAW) || preserveXAttrs) {
|
||||||
|
target.fs.setXAttr(target.path, entry.getKey(), entry.getValue());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -143,7 +143,11 @@ class CopyCommands {
|
||||||
"timestamps, ownership, permission. If -pa is specified, " +
|
"timestamps, ownership, permission. If -pa is specified, " +
|
||||||
"then preserves permission also because ACL is a super-set of " +
|
"then preserves permission also because ACL is a super-set of " +
|
||||||
"permission. Passing -f overwrites the destination if it " +
|
"permission. Passing -f overwrites the destination if it " +
|
||||||
"already exists.\n";
|
"already exists. raw namespace extended attributes are preserved " +
|
||||||
|
"if (1) they are supported (HDFS only) and, (2) all of the source and " +
|
||||||
|
"target pathnames are in the /.reserved/raw hierarchy. raw namespace " +
|
||||||
|
"xattr preservation is determined solely by the presence (or absence) " +
|
||||||
|
"of the /.reserved/raw prefix and not by the -p option.\n";
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void processOptions(LinkedList<String> args) throws IOException {
|
protected void processOptions(LinkedList<String> args) throws IOException {
|
||||||
|
|
|
@ -78,6 +78,11 @@ public class NativeCodeLoader {
|
||||||
* Returns true only if this build was compiled with support for snappy.
|
* Returns true only if this build was compiled with support for snappy.
|
||||||
*/
|
*/
|
||||||
public static native boolean buildSupportsSnappy();
|
public static native boolean buildSupportsSnappy();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns true only if this build was compiled with support for openssl.
|
||||||
|
*/
|
||||||
|
public static native boolean buildSupportsOpenssl();
|
||||||
|
|
||||||
public static native String getLibraryName();
|
public static native String getLibraryName();
|
||||||
|
|
||||||
|
|
|
@ -20,6 +20,7 @@ package org.apache.hadoop.util;
|
||||||
|
|
||||||
import org.apache.hadoop.util.NativeCodeLoader;
|
import org.apache.hadoop.util.NativeCodeLoader;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.crypto.OpensslCipher;
|
||||||
import org.apache.hadoop.io.compress.Lz4Codec;
|
import org.apache.hadoop.io.compress.Lz4Codec;
|
||||||
import org.apache.hadoop.io.compress.SnappyCodec;
|
import org.apache.hadoop.io.compress.SnappyCodec;
|
||||||
import org.apache.hadoop.io.compress.bzip2.Bzip2Factory;
|
import org.apache.hadoop.io.compress.bzip2.Bzip2Factory;
|
||||||
|
@ -60,6 +61,8 @@ public class NativeLibraryChecker {
|
||||||
// lz4 is linked within libhadoop
|
// lz4 is linked within libhadoop
|
||||||
boolean lz4Loaded = nativeHadoopLoaded;
|
boolean lz4Loaded = nativeHadoopLoaded;
|
||||||
boolean bzip2Loaded = Bzip2Factory.isNativeBzip2Loaded(conf);
|
boolean bzip2Loaded = Bzip2Factory.isNativeBzip2Loaded(conf);
|
||||||
|
boolean openSslLoaded = false;
|
||||||
|
String openSslDetail = "";
|
||||||
String hadoopLibraryName = "";
|
String hadoopLibraryName = "";
|
||||||
String zlibLibraryName = "";
|
String zlibLibraryName = "";
|
||||||
String snappyLibraryName = "";
|
String snappyLibraryName = "";
|
||||||
|
@ -76,6 +79,13 @@ public class NativeLibraryChecker {
|
||||||
if (snappyLoaded && NativeCodeLoader.buildSupportsSnappy()) {
|
if (snappyLoaded && NativeCodeLoader.buildSupportsSnappy()) {
|
||||||
snappyLibraryName = SnappyCodec.getLibraryName();
|
snappyLibraryName = SnappyCodec.getLibraryName();
|
||||||
}
|
}
|
||||||
|
if (OpensslCipher.getLoadingFailureReason() != null) {
|
||||||
|
openSslDetail = OpensslCipher.getLoadingFailureReason();
|
||||||
|
openSslLoaded = false;
|
||||||
|
} else {
|
||||||
|
openSslDetail = OpensslCipher.getLibraryName();
|
||||||
|
openSslLoaded = true;
|
||||||
|
}
|
||||||
if (lz4Loaded) {
|
if (lz4Loaded) {
|
||||||
lz4LibraryName = Lz4Codec.getLibraryName();
|
lz4LibraryName = Lz4Codec.getLibraryName();
|
||||||
}
|
}
|
||||||
|
@ -84,11 +94,12 @@ public class NativeLibraryChecker {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
System.out.println("Native library checking:");
|
System.out.println("Native library checking:");
|
||||||
System.out.printf("hadoop: %b %s\n", nativeHadoopLoaded, hadoopLibraryName);
|
System.out.printf("hadoop: %b %s\n", nativeHadoopLoaded, hadoopLibraryName);
|
||||||
System.out.printf("zlib: %b %s\n", zlibLoaded, zlibLibraryName);
|
System.out.printf("zlib: %b %s\n", zlibLoaded, zlibLibraryName);
|
||||||
System.out.printf("snappy: %b %s\n", snappyLoaded, snappyLibraryName);
|
System.out.printf("snappy: %b %s\n", snappyLoaded, snappyLibraryName);
|
||||||
System.out.printf("lz4: %b %s\n", lz4Loaded, lz4LibraryName);
|
System.out.printf("lz4: %b %s\n", lz4Loaded, lz4LibraryName);
|
||||||
System.out.printf("bzip2: %b %s\n", bzip2Loaded, bzip2LibraryName);
|
System.out.printf("bzip2: %b %s\n", bzip2Loaded, bzip2LibraryName);
|
||||||
|
System.out.printf("openssl: %b %s\n", openSslLoaded, openSslDetail);
|
||||||
if ((!nativeHadoopLoaded) ||
|
if ((!nativeHadoopLoaded) ||
|
||||||
(checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded && bzip2Loaded))) {
|
(checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded && bzip2Loaded))) {
|
||||||
// return 1 to indicated check failed
|
// return 1 to indicated check failed
|
||||||
|
|
|
@ -0,0 +1,382 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include "org_apache_hadoop_crypto.h"
|
||||||
|
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
#include "org_apache_hadoop_crypto_OpensslCipher.h"
|
||||||
|
|
||||||
|
#ifdef UNIX
|
||||||
|
static EVP_CIPHER_CTX * (*dlsym_EVP_CIPHER_CTX_new)(void);
|
||||||
|
static void (*dlsym_EVP_CIPHER_CTX_free)(EVP_CIPHER_CTX *);
|
||||||
|
static int (*dlsym_EVP_CIPHER_CTX_cleanup)(EVP_CIPHER_CTX *);
|
||||||
|
static void (*dlsym_EVP_CIPHER_CTX_init)(EVP_CIPHER_CTX *);
|
||||||
|
static int (*dlsym_EVP_CIPHER_CTX_set_padding)(EVP_CIPHER_CTX *, int);
|
||||||
|
static int (*dlsym_EVP_CipherInit_ex)(EVP_CIPHER_CTX *, const EVP_CIPHER *, \
|
||||||
|
ENGINE *, const unsigned char *, const unsigned char *, int);
|
||||||
|
static int (*dlsym_EVP_CipherUpdate)(EVP_CIPHER_CTX *, unsigned char *, \
|
||||||
|
int *, const unsigned char *, int);
|
||||||
|
static int (*dlsym_EVP_CipherFinal_ex)(EVP_CIPHER_CTX *, unsigned char *, int *);
|
||||||
|
static EVP_CIPHER * (*dlsym_EVP_aes_256_ctr)(void);
|
||||||
|
static EVP_CIPHER * (*dlsym_EVP_aes_128_ctr)(void);
|
||||||
|
static void *openssl;
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifdef WINDOWS
|
||||||
|
typedef EVP_CIPHER_CTX * (__cdecl *__dlsym_EVP_CIPHER_CTX_new)(void);
|
||||||
|
typedef void (__cdecl *__dlsym_EVP_CIPHER_CTX_free)(EVP_CIPHER_CTX *);
|
||||||
|
typedef int (__cdecl *__dlsym_EVP_CIPHER_CTX_cleanup)(EVP_CIPHER_CTX *);
|
||||||
|
typedef void (__cdecl *__dlsym_EVP_CIPHER_CTX_init)(EVP_CIPHER_CTX *);
|
||||||
|
typedef int (__cdecl *__dlsym_EVP_CIPHER_CTX_set_padding)(EVP_CIPHER_CTX *, int);
|
||||||
|
typedef int (__cdecl *__dlsym_EVP_CipherInit_ex)(EVP_CIPHER_CTX *, \
|
||||||
|
const EVP_CIPHER *, ENGINE *, const unsigned char *, \
|
||||||
|
const unsigned char *, int);
|
||||||
|
typedef int (__cdecl *__dlsym_EVP_CipherUpdate)(EVP_CIPHER_CTX *, \
|
||||||
|
unsigned char *, int *, const unsigned char *, int);
|
||||||
|
typedef int (__cdecl *__dlsym_EVP_CipherFinal_ex)(EVP_CIPHER_CTX *, \
|
||||||
|
unsigned char *, int *);
|
||||||
|
typedef EVP_CIPHER * (__cdecl *__dlsym_EVP_aes_256_ctr)(void);
|
||||||
|
typedef EVP_CIPHER * (__cdecl *__dlsym_EVP_aes_128_ctr)(void);
|
||||||
|
static __dlsym_EVP_CIPHER_CTX_new dlsym_EVP_CIPHER_CTX_new;
|
||||||
|
static __dlsym_EVP_CIPHER_CTX_free dlsym_EVP_CIPHER_CTX_free;
|
||||||
|
static __dlsym_EVP_CIPHER_CTX_cleanup dlsym_EVP_CIPHER_CTX_cleanup;
|
||||||
|
static __dlsym_EVP_CIPHER_CTX_init dlsym_EVP_CIPHER_CTX_init;
|
||||||
|
static __dlsym_EVP_CIPHER_CTX_set_padding dlsym_EVP_CIPHER_CTX_set_padding;
|
||||||
|
static __dlsym_EVP_CipherInit_ex dlsym_EVP_CipherInit_ex;
|
||||||
|
static __dlsym_EVP_CipherUpdate dlsym_EVP_CipherUpdate;
|
||||||
|
static __dlsym_EVP_CipherFinal_ex dlsym_EVP_CipherFinal_ex;
|
||||||
|
static __dlsym_EVP_aes_256_ctr dlsym_EVP_aes_256_ctr;
|
||||||
|
static __dlsym_EVP_aes_128_ctr dlsym_EVP_aes_128_ctr;
|
||||||
|
static HMODULE openssl;
|
||||||
|
#endif
|
||||||
|
|
||||||
|
static void loadAesCtr(JNIEnv *env)
|
||||||
|
{
|
||||||
|
#ifdef UNIX
|
||||||
|
LOAD_DYNAMIC_SYMBOL(dlsym_EVP_aes_256_ctr, env, openssl, "EVP_aes_256_ctr");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(dlsym_EVP_aes_128_ctr, env, openssl, "EVP_aes_128_ctr");
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifdef WINDOWS
|
||||||
|
LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_aes_256_ctr, dlsym_EVP_aes_256_ctr, \
|
||||||
|
env, openssl, "EVP_aes_256_ctr");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_aes_128_ctr, dlsym_EVP_aes_128_ctr, \
|
||||||
|
env, openssl, "EVP_aes_128_ctr");
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs
|
||||||
|
(JNIEnv *env, jclass clazz)
|
||||||
|
{
|
||||||
|
char msg[1000];
|
||||||
|
#ifdef UNIX
|
||||||
|
openssl = dlopen(HADOOP_OPENSSL_LIBRARY, RTLD_LAZY | RTLD_GLOBAL);
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifdef WINDOWS
|
||||||
|
openssl = LoadLibrary(HADOOP_OPENSSL_LIBRARY);
|
||||||
|
#endif
|
||||||
|
|
||||||
|
if (!openssl) {
|
||||||
|
snprintf(msg, sizeof(msg), "Cannot load %s (%s)!", HADOOP_OPENSSL_LIBRARY, \
|
||||||
|
dlerror());
|
||||||
|
THROW(env, "java/lang/UnsatisfiedLinkError", msg);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
#ifdef UNIX
|
||||||
|
dlerror(); // Clear any existing error
|
||||||
|
LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_new, env, openssl, \
|
||||||
|
"EVP_CIPHER_CTX_new");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_free, env, openssl, \
|
||||||
|
"EVP_CIPHER_CTX_free");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_cleanup, env, openssl, \
|
||||||
|
"EVP_CIPHER_CTX_cleanup");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_init, env, openssl, \
|
||||||
|
"EVP_CIPHER_CTX_init");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_set_padding, env, openssl, \
|
||||||
|
"EVP_CIPHER_CTX_set_padding");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CipherInit_ex, env, openssl, \
|
||||||
|
"EVP_CipherInit_ex");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CipherUpdate, env, openssl, \
|
||||||
|
"EVP_CipherUpdate");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CipherFinal_ex, env, openssl, \
|
||||||
|
"EVP_CipherFinal_ex");
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifdef WINDOWS
|
||||||
|
LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_new, dlsym_EVP_CIPHER_CTX_new, \
|
||||||
|
env, openssl, "EVP_CIPHER_CTX_new");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_free, dlsym_EVP_CIPHER_CTX_free, \
|
||||||
|
env, openssl, "EVP_CIPHER_CTX_free");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_cleanup, \
|
||||||
|
dlsym_EVP_CIPHER_CTX_cleanup, env,
|
||||||
|
openssl, "EVP_CIPHER_CTX_cleanup");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_init, dlsym_EVP_CIPHER_CTX_init, \
|
||||||
|
env, openssl, "EVP_CIPHER_CTX_init");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_set_padding, \
|
||||||
|
dlsym_EVP_CIPHER_CTX_set_padding, env, \
|
||||||
|
openssl, "EVP_CIPHER_CTX_set_padding");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CipherInit_ex, dlsym_EVP_CipherInit_ex, \
|
||||||
|
env, openssl, "EVP_CipherInit_ex");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CipherUpdate, dlsym_EVP_CipherUpdate, \
|
||||||
|
env, openssl, "EVP_CipherUpdate");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CipherFinal_ex, dlsym_EVP_CipherFinal_ex, \
|
||||||
|
env, openssl, "EVP_CipherFinal_ex");
|
||||||
|
#endif
|
||||||
|
|
||||||
|
loadAesCtr(env);
|
||||||
|
jthrowable jthr = (*env)->ExceptionOccurred(env);
|
||||||
|
if (jthr) {
|
||||||
|
(*env)->DeleteLocalRef(env, jthr);
|
||||||
|
THROW(env, "java/lang/UnsatisfiedLinkError", \
|
||||||
|
"Cannot find AES-CTR support, is your version of Openssl new enough?");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
JNIEXPORT jlong JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initContext
|
||||||
|
(JNIEnv *env, jclass clazz, jint alg, jint padding)
|
||||||
|
{
|
||||||
|
if (alg != AES_CTR) {
|
||||||
|
THROW(env, "java/security/NoSuchAlgorithmException", NULL);
|
||||||
|
return (jlong)0;
|
||||||
|
}
|
||||||
|
if (padding != NOPADDING) {
|
||||||
|
THROW(env, "javax/crypto/NoSuchPaddingException", NULL);
|
||||||
|
return (jlong)0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (dlsym_EVP_aes_256_ctr == NULL || dlsym_EVP_aes_128_ctr == NULL) {
|
||||||
|
THROW(env, "java/security/NoSuchAlgorithmException", \
|
||||||
|
"Doesn't support AES CTR.");
|
||||||
|
return (jlong)0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create and initialize a EVP_CIPHER_CTX
|
||||||
|
EVP_CIPHER_CTX *context = dlsym_EVP_CIPHER_CTX_new();
|
||||||
|
if (!context) {
|
||||||
|
THROW(env, "java/lang/OutOfMemoryError", NULL);
|
||||||
|
return (jlong)0;
|
||||||
|
}
|
||||||
|
|
||||||
|
return JLONG(context);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only supports AES-CTR currently
|
||||||
|
static EVP_CIPHER * getEvpCipher(int alg, int keyLen)
|
||||||
|
{
|
||||||
|
EVP_CIPHER *cipher = NULL;
|
||||||
|
if (alg == AES_CTR) {
|
||||||
|
if (keyLen == KEY_LENGTH_256) {
|
||||||
|
cipher = dlsym_EVP_aes_256_ctr();
|
||||||
|
} else if (keyLen == KEY_LENGTH_128) {
|
||||||
|
cipher = dlsym_EVP_aes_128_ctr();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return cipher;
|
||||||
|
}
|
||||||
|
|
||||||
|
JNIEXPORT jlong JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_init
|
||||||
|
(JNIEnv *env, jobject object, jlong ctx, jint mode, jint alg, jint padding,
|
||||||
|
jbyteArray key, jbyteArray iv)
|
||||||
|
{
|
||||||
|
int jKeyLen = (*env)->GetArrayLength(env, key);
|
||||||
|
int jIvLen = (*env)->GetArrayLength(env, iv);
|
||||||
|
if (jKeyLen != KEY_LENGTH_128 && jKeyLen != KEY_LENGTH_256) {
|
||||||
|
THROW(env, "java/lang/IllegalArgumentException", "Invalid key length.");
|
||||||
|
return (jlong)0;
|
||||||
|
}
|
||||||
|
if (jIvLen != IV_LENGTH) {
|
||||||
|
THROW(env, "java/lang/IllegalArgumentException", "Invalid iv length.");
|
||||||
|
return (jlong)0;
|
||||||
|
}
|
||||||
|
|
||||||
|
EVP_CIPHER_CTX *context = CONTEXT(ctx);
|
||||||
|
if (context == 0) {
|
||||||
|
// Create and initialize a EVP_CIPHER_CTX
|
||||||
|
context = dlsym_EVP_CIPHER_CTX_new();
|
||||||
|
if (!context) {
|
||||||
|
THROW(env, "java/lang/OutOfMemoryError", NULL);
|
||||||
|
return (jlong)0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
jbyte *jKey = (*env)->GetByteArrayElements(env, key, NULL);
|
||||||
|
if (jKey == NULL) {
|
||||||
|
THROW(env, "java/lang/InternalError", "Cannot get bytes array for key.");
|
||||||
|
return (jlong)0;
|
||||||
|
}
|
||||||
|
jbyte *jIv = (*env)->GetByteArrayElements(env, iv, NULL);
|
||||||
|
if (jIv == NULL) {
|
||||||
|
(*env)->ReleaseByteArrayElements(env, key, jKey, 0);
|
||||||
|
THROW(env, "java/lang/InternalError", "Cannot get bytes array for iv.");
|
||||||
|
return (jlong)0;
|
||||||
|
}
|
||||||
|
|
||||||
|
int rc = dlsym_EVP_CipherInit_ex(context, getEvpCipher(alg, jKeyLen), \
|
||||||
|
NULL, (unsigned char *)jKey, (unsigned char *)jIv, mode == ENCRYPT_MODE);
|
||||||
|
(*env)->ReleaseByteArrayElements(env, key, jKey, 0);
|
||||||
|
(*env)->ReleaseByteArrayElements(env, iv, jIv, 0);
|
||||||
|
if (rc == 0) {
|
||||||
|
dlsym_EVP_CIPHER_CTX_cleanup(context);
|
||||||
|
THROW(env, "java/lang/InternalError", "Error in EVP_CipherInit_ex.");
|
||||||
|
return (jlong)0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (padding == NOPADDING) {
|
||||||
|
dlsym_EVP_CIPHER_CTX_set_padding(context, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
return JLONG(context);
|
||||||
|
}
|
||||||
|
|
||||||
|
// https://www.openssl.org/docs/crypto/EVP_EncryptInit.html
|
||||||
|
static int check_update_max_output_len(EVP_CIPHER_CTX *context, int input_len,
|
||||||
|
int max_output_len)
|
||||||
|
{
|
||||||
|
if (context->flags & EVP_CIPH_NO_PADDING) {
|
||||||
|
if (max_output_len >= input_len) {
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
} else {
|
||||||
|
int b = context->cipher->block_size;
|
||||||
|
if (context->encrypt) {
|
||||||
|
if (max_output_len >= input_len + b - 1) {
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (max_output_len >= input_len + b) {
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
JNIEXPORT jint JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_update
|
||||||
|
(JNIEnv *env, jobject object, jlong ctx, jobject input, jint input_offset,
|
||||||
|
jint input_len, jobject output, jint output_offset, jint max_output_len)
|
||||||
|
{
|
||||||
|
EVP_CIPHER_CTX *context = CONTEXT(ctx);
|
||||||
|
if (!check_update_max_output_len(context, input_len, max_output_len)) {
|
||||||
|
THROW(env, "javax/crypto/ShortBufferException", \
|
||||||
|
"Output buffer is not sufficient.");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
unsigned char *input_bytes = (*env)->GetDirectBufferAddress(env, input);
|
||||||
|
unsigned char *output_bytes = (*env)->GetDirectBufferAddress(env, output);
|
||||||
|
if (input_bytes == NULL || output_bytes == NULL) {
|
||||||
|
THROW(env, "java/lang/InternalError", "Cannot get buffer address.");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
input_bytes = input_bytes + input_offset;
|
||||||
|
output_bytes = output_bytes + output_offset;
|
||||||
|
|
||||||
|
int output_len = 0;
|
||||||
|
if (!dlsym_EVP_CipherUpdate(context, output_bytes, &output_len, \
|
||||||
|
input_bytes, input_len)) {
|
||||||
|
dlsym_EVP_CIPHER_CTX_cleanup(context);
|
||||||
|
THROW(env, "java/lang/InternalError", "Error in EVP_CipherUpdate.");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
return output_len;
|
||||||
|
}
|
||||||
|
|
||||||
|
// https://www.openssl.org/docs/crypto/EVP_EncryptInit.html
|
||||||
|
static int check_doFinal_max_output_len(EVP_CIPHER_CTX *context,
|
||||||
|
int max_output_len)
|
||||||
|
{
|
||||||
|
if (context->flags & EVP_CIPH_NO_PADDING) {
|
||||||
|
return 1;
|
||||||
|
} else {
|
||||||
|
int b = context->cipher->block_size;
|
||||||
|
if (max_output_len >= b) {
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
JNIEXPORT jint JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_doFinal
|
||||||
|
(JNIEnv *env, jobject object, jlong ctx, jobject output, jint offset,
|
||||||
|
jint max_output_len)
|
||||||
|
{
|
||||||
|
EVP_CIPHER_CTX *context = CONTEXT(ctx);
|
||||||
|
if (!check_doFinal_max_output_len(context, max_output_len)) {
|
||||||
|
THROW(env, "javax/crypto/ShortBufferException", \
|
||||||
|
"Output buffer is not sufficient.");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
unsigned char *output_bytes = (*env)->GetDirectBufferAddress(env, output);
|
||||||
|
if (output_bytes == NULL) {
|
||||||
|
THROW(env, "java/lang/InternalError", "Cannot get buffer address.");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
output_bytes = output_bytes + offset;
|
||||||
|
|
||||||
|
int output_len = 0;
|
||||||
|
if (!dlsym_EVP_CipherFinal_ex(context, output_bytes, &output_len)) {
|
||||||
|
dlsym_EVP_CIPHER_CTX_cleanup(context);
|
||||||
|
THROW(env, "java/lang/InternalError", "Error in EVP_CipherFinal_ex.");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
return output_len;
|
||||||
|
}
|
||||||
|
|
||||||
|
JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_clean
|
||||||
|
(JNIEnv *env, jobject object, jlong ctx)
|
||||||
|
{
|
||||||
|
EVP_CIPHER_CTX *context = CONTEXT(ctx);
|
||||||
|
if (context) {
|
||||||
|
dlsym_EVP_CIPHER_CTX_free(context);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
JNIEXPORT jstring JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_getLibraryName
|
||||||
|
(JNIEnv *env, jclass clazz)
|
||||||
|
{
|
||||||
|
#ifdef UNIX
|
||||||
|
if (dlsym_EVP_CIPHER_CTX_init) {
|
||||||
|
Dl_info dl_info;
|
||||||
|
if(dladdr(
|
||||||
|
dlsym_EVP_CIPHER_CTX_init,
|
||||||
|
&dl_info)) {
|
||||||
|
return (*env)->NewStringUTF(env, dl_info.dli_fname);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return (*env)->NewStringUTF(env, HADOOP_OPENSSL_LIBRARY);
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifdef WINDOWS
|
||||||
|
LPWSTR filename = NULL;
|
||||||
|
GetLibraryName(dlsym_EVP_CIPHER_CTX_init, &filename);
|
||||||
|
if (filename != NULL) {
|
||||||
|
return (*env)->NewString(env, filename, (jsize) wcslen(filename));
|
||||||
|
} else {
|
||||||
|
return (*env)->NewStringUTF(env, "Unavailable");
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
}
|
|
@ -0,0 +1,61 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#ifndef ORG_APACHE_HADOOP_CRYPTO_H
|
||||||
|
#define ORG_APACHE_HADOOP_CRYPTO_H
|
||||||
|
|
||||||
|
#include "org_apache_hadoop.h"
|
||||||
|
|
||||||
|
#ifdef UNIX
|
||||||
|
#include <dlfcn.h>
|
||||||
|
#include "config.h"
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifdef WINDOWS
|
||||||
|
#include "winutils.h"
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#include <openssl/aes.h>
|
||||||
|
#include <openssl/evp.h>
|
||||||
|
#include <openssl/err.h>
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A helper macro to convert the java 'context-handle'
|
||||||
|
* to a EVP_CIPHER_CTX pointer.
|
||||||
|
*/
|
||||||
|
#define CONTEXT(context) ((EVP_CIPHER_CTX*)((ptrdiff_t)(context)))
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A helper macro to convert the EVP_CIPHER_CTX pointer to the
|
||||||
|
* java 'context-handle'.
|
||||||
|
*/
|
||||||
|
#define JLONG(context) ((jlong)((ptrdiff_t)(context)))
|
||||||
|
|
||||||
|
#define KEY_LENGTH_128 16
|
||||||
|
#define KEY_LENGTH_256 32
|
||||||
|
#define IV_LENGTH 16
|
||||||
|
|
||||||
|
#define ENCRYPT_MODE 1
|
||||||
|
#define DECRYPT_MODE 0
|
||||||
|
|
||||||
|
/** Currently only support AES/CTR/NoPadding. */
|
||||||
|
#define AES_CTR 0
|
||||||
|
#define NOPADDING 0
|
||||||
|
#define PKCSPADDING 1
|
||||||
|
|
||||||
|
#endif //ORG_APACHE_HADOOP_CRYPTO_H
|
|
@ -0,0 +1,335 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include "org_apache_hadoop_crypto_random.h"
|
||||||
|
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
#ifdef UNIX
|
||||||
|
#include <pthread.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
#include <sys/syscall.h>
|
||||||
|
#include <sys/types.h>
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifdef WINDOWS
|
||||||
|
#include <windows.h>
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#include "org_apache_hadoop_crypto_random_OpensslSecureRandom.h"
|
||||||
|
|
||||||
|
#ifdef UNIX
|
||||||
|
static void * (*dlsym_CRYPTO_malloc) (int, const char *, int);
|
||||||
|
static void (*dlsym_CRYPTO_free) (void *);
|
||||||
|
static int (*dlsym_CRYPTO_num_locks) (void);
|
||||||
|
static void (*dlsym_CRYPTO_set_locking_callback) (void (*)());
|
||||||
|
static void (*dlsym_CRYPTO_set_id_callback) (unsigned long (*)());
|
||||||
|
static void (*dlsym_ENGINE_load_rdrand) (void);
|
||||||
|
static ENGINE * (*dlsym_ENGINE_by_id) (const char *);
|
||||||
|
static int (*dlsym_ENGINE_init) (ENGINE *);
|
||||||
|
static int (*dlsym_ENGINE_set_default) (ENGINE *, unsigned int);
|
||||||
|
static int (*dlsym_ENGINE_finish) (ENGINE *);
|
||||||
|
static int (*dlsym_ENGINE_free) (ENGINE *);
|
||||||
|
static void (*dlsym_ENGINE_cleanup) (void);
|
||||||
|
static int (*dlsym_RAND_bytes) (unsigned char *, int);
|
||||||
|
static unsigned long (*dlsym_ERR_get_error) (void);
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifdef WINDOWS
|
||||||
|
typedef void * (__cdecl *__dlsym_CRYPTO_malloc) (int, const char *, int);
|
||||||
|
typedef void (__cdecl *__dlsym_CRYPTO_free) (void *);
|
||||||
|
typedef int (__cdecl *__dlsym_CRYPTO_num_locks) (void);
|
||||||
|
typedef void (__cdecl *__dlsym_CRYPTO_set_locking_callback) \
|
||||||
|
(void (*)(int, int, char *, int);
|
||||||
|
typedef void (__cdecl *__dlsym_ENGINE_load_rdrand) (void);
|
||||||
|
typedef ENGINE * (__cdecl *__dlsym_ENGINE_by_id) (const char *);
|
||||||
|
typedef int (__cdecl *__dlsym_ENGINE_init) (ENGINE *);
|
||||||
|
typedef int (__cdecl *__dlsym_ENGINE_set_default) (ENGINE *, unsigned int);
|
||||||
|
typedef int (__cdecl *__dlsym_ENGINE_finish) (ENGINE *);
|
||||||
|
typedef int (__cdecl *__dlsym_ENGINE_free) (ENGINE *);
|
||||||
|
typedef void (__cdecl *__dlsym_ENGINE_cleanup) (void);
|
||||||
|
typedef int (__cdecl *__dlsym_RAND_bytes) (unsigned char *, int);
|
||||||
|
typedef unsigned long (__cdecl *__dlsym_ERR_get_error) (void);
|
||||||
|
static __dlsym_CRYPTO_malloc dlsym_CRYPTO_malloc;
|
||||||
|
static __dlsym_CRYPTO_free dlsym_CRYPTO_free;
|
||||||
|
static __dlsym_CRYPTO_num_locks dlsym_CRYPTO_num_locks;
|
||||||
|
static __dlsym_CRYPTO_set_locking_callback dlsym_CRYPTO_set_locking_callback;
|
||||||
|
static __dlsym_ENGINE_load_rdrand dlsym_ENGINE_load_rdrand;
|
||||||
|
static __dlsym_ENGINE_by_id dlsym_ENGINE_by_id;
|
||||||
|
static __dlsym_ENGINE_init dlsym_ENGINE_init;
|
||||||
|
static __dlsym_ENGINE_set_default dlsym_ENGINE_set_default;
|
||||||
|
static __dlsym_ENGINE_finish dlsym_ENGINE_finish;
|
||||||
|
static __dlsym_ENGINE_free dlsym_ENGINE_free;
|
||||||
|
static __dlsym_ENGINE_cleanup dlsym_ENGINE_cleanup;
|
||||||
|
static __dlsym_RAND_bytes dlsym_RAND_bytes;
|
||||||
|
static __dlsym_ERR_get_error dlsym_ERR_get_error;
|
||||||
|
#endif
|
||||||
|
|
||||||
|
static ENGINE * openssl_rand_init(void);
|
||||||
|
static void openssl_rand_clean(ENGINE *eng, int clean_locks);
|
||||||
|
static int openssl_rand_bytes(unsigned char *buf, int num);
|
||||||
|
|
||||||
|
JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_initSR
|
||||||
|
(JNIEnv *env, jclass clazz)
|
||||||
|
{
|
||||||
|
char msg[1000];
|
||||||
|
#ifdef UNIX
|
||||||
|
void *openssl = dlopen(HADOOP_OPENSSL_LIBRARY, RTLD_LAZY | RTLD_GLOBAL);
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifdef WINDOWS
|
||||||
|
HMODULE openssl = LoadLibrary(HADOOP_OPENSSL_LIBRARY);
|
||||||
|
#endif
|
||||||
|
|
||||||
|
if (!openssl) {
|
||||||
|
snprintf(msg, sizeof(msg), "Cannot load %s (%s)!", HADOOP_OPENSSL_LIBRARY, \
|
||||||
|
dlerror());
|
||||||
|
THROW(env, "java/lang/UnsatisfiedLinkError", msg);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
#ifdef UNIX
|
||||||
|
dlerror(); // Clear any existing error
|
||||||
|
LOAD_DYNAMIC_SYMBOL(dlsym_CRYPTO_malloc, env, openssl, "CRYPTO_malloc");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(dlsym_CRYPTO_free, env, openssl, "CRYPTO_free");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(dlsym_CRYPTO_num_locks, env, openssl, "CRYPTO_num_locks");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(dlsym_CRYPTO_set_locking_callback, \
|
||||||
|
env, openssl, "CRYPTO_set_locking_callback");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(dlsym_CRYPTO_set_id_callback, env, \
|
||||||
|
openssl, "CRYPTO_set_id_callback");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_load_rdrand, env, \
|
||||||
|
openssl, "ENGINE_load_rdrand");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_by_id, env, openssl, "ENGINE_by_id");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_init, env, openssl, "ENGINE_init");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_set_default, env, \
|
||||||
|
openssl, "ENGINE_set_default");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_finish, env, openssl, "ENGINE_finish");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_free, env, openssl, "ENGINE_free");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_cleanup, env, openssl, "ENGINE_cleanup");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(dlsym_RAND_bytes, env, openssl, "RAND_bytes");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(dlsym_ERR_get_error, env, openssl, "ERR_get_error");
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifdef WINDOWS
|
||||||
|
LOAD_DYNAMIC_SYMBOL(__dlsym_CRYPTO_malloc, dlsym_CRYPTO_malloc, \
|
||||||
|
env, openssl, "CRYPTO_malloc");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(__dlsym_CRYPTO_free, dlsym_CRYPTO_free, \
|
||||||
|
env, openssl, "CRYPTO_free");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(__dlsym_CRYPTO_num_locks, dlsym_CRYPTO_num_locks, \
|
||||||
|
env, openssl, "CRYPTO_num_locks");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(__dlsym_CRYPTO_set_locking_callback, \
|
||||||
|
dlsym_CRYPTO_set_locking_callback, \
|
||||||
|
env, openssl, "CRYPTO_set_locking_callback");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(__dlsym_ENGINE_load_rdrand, dlsym_ENGINE_load_rdrand, \
|
||||||
|
env, openssl, "ENGINE_load_rdrand");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(__dlsym_ENGINE_by_id, dlsym_ENGINE_by_id, \
|
||||||
|
env, openssl, "ENGINE_by_id");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(__dlsym_ENGINE_init, dlsym_ENGINE_init, \
|
||||||
|
env, openssl, "ENGINE_init");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(__dlsym_ENGINE_set_default, dlsym_ENGINE_set_default, \
|
||||||
|
env, openssl, "ENGINE_set_default");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(__dlsym_ENGINE_finish, dlsym_ENGINE_finish, \
|
||||||
|
env, openssl, "ENGINE_finish");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(__dlsym_ENGINE_free, dlsym_ENGINE_free, \
|
||||||
|
env, openssl, "ENGINE_free");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(__dlsym_ENGINE_cleanup, dlsym_ENGINE_cleanup, \
|
||||||
|
env, openssl, "ENGINE_cleanup");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(__dlsym_RAND_bytes, dlsym_RAND_bytes, \
|
||||||
|
env, openssl, "RAND_bytes");
|
||||||
|
LOAD_DYNAMIC_SYMBOL(__dlsym_ERR_get_error, dlsym_ERR_get_error, \
|
||||||
|
env, openssl, "ERR_get_error");
|
||||||
|
#endif
|
||||||
|
|
||||||
|
openssl_rand_init();
|
||||||
|
}
|
||||||
|
|
||||||
|
JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_nextRandBytes___3B
|
||||||
|
(JNIEnv *env, jobject object, jbyteArray bytes)
|
||||||
|
{
|
||||||
|
if (NULL == bytes) {
|
||||||
|
THROW(env, "java/lang/NullPointerException", "Buffer cannot be null.");
|
||||||
|
return JNI_FALSE;
|
||||||
|
}
|
||||||
|
jbyte *b = (*env)->GetByteArrayElements(env, bytes, NULL);
|
||||||
|
if (NULL == b) {
|
||||||
|
THROW(env, "java/lang/InternalError", "Cannot get bytes array.");
|
||||||
|
return JNI_FALSE;
|
||||||
|
}
|
||||||
|
int b_len = (*env)->GetArrayLength(env, bytes);
|
||||||
|
int ret = openssl_rand_bytes((unsigned char *)b, b_len);
|
||||||
|
(*env)->ReleaseByteArrayElements(env, bytes, b, 0);
|
||||||
|
|
||||||
|
if (1 != ret) {
|
||||||
|
return JNI_FALSE;
|
||||||
|
}
|
||||||
|
return JNI_TRUE;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* To ensure thread safety for random number generators, we need to call
|
||||||
|
* CRYPTO_set_locking_callback.
|
||||||
|
* http://wiki.openssl.org/index.php/Random_Numbers
|
||||||
|
* Example: crypto/threads/mttest.c
|
||||||
|
*/
|
||||||
|
|
||||||
|
#ifdef WINDOWS
|
||||||
|
static void windows_locking_callback(int mode, int type, char *file, int line);
|
||||||
|
static HANDLE *lock_cs;
|
||||||
|
|
||||||
|
static void locks_setup(void)
|
||||||
|
{
|
||||||
|
int i;
|
||||||
|
lock_cs = dlsym_CRYPTO_malloc(dlsym_CRYPTO_num_locks() * sizeof(HANDLE), \
|
||||||
|
__FILE__, __LINE__);
|
||||||
|
|
||||||
|
for (i = 0; i < dlsym_CRYPTO_num_locks(); i++) {
|
||||||
|
lock_cs[i] = CreateMutex(NULL, FALSE, NULL);
|
||||||
|
}
|
||||||
|
dlsym_CRYPTO_set_locking_callback((void (*)(int, int, char *, int)) \
|
||||||
|
windows_locking_callback);
|
||||||
|
/* id callback defined */
|
||||||
|
}
|
||||||
|
|
||||||
|
static void locks_cleanup(void)
|
||||||
|
{
|
||||||
|
int i;
|
||||||
|
dlsym_CRYPTO_set_locking_callback(NULL);
|
||||||
|
|
||||||
|
for (i = 0; i < dlsym_CRYPTO_num_locks(); i++) {
|
||||||
|
CloseHandle(lock_cs[i]);
|
||||||
|
}
|
||||||
|
dlsym_CRYPTO_free(lock_cs);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void windows_locking_callback(int mode, int type, char *file, int line)
|
||||||
|
{
|
||||||
|
UNUSED(file), UNUSED(line);
|
||||||
|
|
||||||
|
if (mode & CRYPTO_LOCK) {
|
||||||
|
WaitForSingleObject(lock_cs[type], INFINITE);
|
||||||
|
} else {
|
||||||
|
ReleaseMutex(lock_cs[type]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#endif /* WINDOWS */
|
||||||
|
|
||||||
|
#ifdef UNIX
|
||||||
|
static void pthreads_locking_callback(int mode, int type, char *file, int line);
|
||||||
|
static unsigned long pthreads_thread_id(void);
|
||||||
|
static pthread_mutex_t *lock_cs;
|
||||||
|
|
||||||
|
static void locks_setup(void)
|
||||||
|
{
|
||||||
|
int i;
|
||||||
|
lock_cs = dlsym_CRYPTO_malloc(dlsym_CRYPTO_num_locks() * \
|
||||||
|
sizeof(pthread_mutex_t), __FILE__, __LINE__);
|
||||||
|
|
||||||
|
for (i = 0; i < dlsym_CRYPTO_num_locks(); i++) {
|
||||||
|
pthread_mutex_init(&(lock_cs[i]), NULL);
|
||||||
|
}
|
||||||
|
|
||||||
|
dlsym_CRYPTO_set_id_callback((unsigned long (*)())pthreads_thread_id);
|
||||||
|
dlsym_CRYPTO_set_locking_callback((void (*)())pthreads_locking_callback);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void locks_cleanup(void)
|
||||||
|
{
|
||||||
|
int i;
|
||||||
|
dlsym_CRYPTO_set_locking_callback(NULL);
|
||||||
|
|
||||||
|
for (i = 0; i < dlsym_CRYPTO_num_locks(); i++) {
|
||||||
|
pthread_mutex_destroy(&(lock_cs[i]));
|
||||||
|
}
|
||||||
|
|
||||||
|
dlsym_CRYPTO_free(lock_cs);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void pthreads_locking_callback(int mode, int type, char *file, int line)
|
||||||
|
{
|
||||||
|
UNUSED(file), UNUSED(line);
|
||||||
|
|
||||||
|
if (mode & CRYPTO_LOCK) {
|
||||||
|
pthread_mutex_lock(&(lock_cs[type]));
|
||||||
|
} else {
|
||||||
|
pthread_mutex_unlock(&(lock_cs[type]));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static unsigned long pthreads_thread_id(void)
|
||||||
|
{
|
||||||
|
return (unsigned long)syscall(SYS_gettid);
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif /* UNIX */
|
||||||
|
|
||||||
|
/**
|
||||||
|
* If using an Intel chipset with RDRAND, the high-performance hardware
|
||||||
|
* random number generator will be used.
|
||||||
|
*/
|
||||||
|
static ENGINE * openssl_rand_init(void)
|
||||||
|
{
|
||||||
|
locks_setup();
|
||||||
|
|
||||||
|
dlsym_ENGINE_load_rdrand();
|
||||||
|
ENGINE *eng = dlsym_ENGINE_by_id("rdrand");
|
||||||
|
|
||||||
|
int ret = -1;
|
||||||
|
do {
|
||||||
|
if (NULL == eng) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
int rc = dlsym_ENGINE_init(eng);
|
||||||
|
if (0 == rc) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
rc = dlsym_ENGINE_set_default(eng, ENGINE_METHOD_RAND);
|
||||||
|
if (0 == rc) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
ret = 0;
|
||||||
|
} while(0);
|
||||||
|
|
||||||
|
if (ret == -1) {
|
||||||
|
openssl_rand_clean(eng, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
return eng;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void openssl_rand_clean(ENGINE *eng, int clean_locks)
|
||||||
|
{
|
||||||
|
if (NULL != eng) {
|
||||||
|
dlsym_ENGINE_finish(eng);
|
||||||
|
dlsym_ENGINE_free(eng);
|
||||||
|
}
|
||||||
|
|
||||||
|
dlsym_ENGINE_cleanup();
|
||||||
|
if (clean_locks) {
|
||||||
|
locks_cleanup();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static int openssl_rand_bytes(unsigned char *buf, int num)
|
||||||
|
{
|
||||||
|
return dlsym_RAND_bytes(buf, num);
|
||||||
|
}
|
|
@ -0,0 +1,40 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#ifndef ORG_APACHE_HADOOP_CRYPTO_RANDOM_H
|
||||||
|
#define ORG_APACHE_HADOOP_CRYPTO_RANDOM_H
|
||||||
|
|
||||||
|
#include "org_apache_hadoop.h"
|
||||||
|
|
||||||
|
#ifdef UNIX
|
||||||
|
#include <dlfcn.h>
|
||||||
|
#include "config.h"
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifdef WINDOWS
|
||||||
|
#include "winutils.h"
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#define UNUSED(x) ((void)(x))
|
||||||
|
|
||||||
|
#include <openssl/crypto.h>
|
||||||
|
#include <openssl/engine.h>
|
||||||
|
#include <openssl/rand.h>
|
||||||
|
#include <openssl/err.h>
|
||||||
|
|
||||||
|
#endif //ORG_APACHE_HADOOP_CRYPTO_RANDOM_H
|
|
@ -39,6 +39,16 @@ JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_buildSup
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_buildSupportsOpenssl
|
||||||
|
(JNIEnv *env, jclass clazz)
|
||||||
|
{
|
||||||
|
#ifdef HADOOP_OPENSSL_LIBRARY
|
||||||
|
return JNI_TRUE;
|
||||||
|
#else
|
||||||
|
return JNI_FALSE;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
JNIEXPORT jstring JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_getLibraryName
|
JNIEXPORT jstring JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_getLibraryName
|
||||||
(JNIEnv *env, jclass clazz)
|
(JNIEnv *env, jclass clazz)
|
||||||
{
|
{
|
||||||
|
|
|
@ -1445,6 +1445,74 @@ for ldap providers in the same way as above does.
|
||||||
true.
|
true.
|
||||||
</description>
|
</description>
|
||||||
</property>
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>hadoop.security.crypto.codec.classes.EXAMPLECIPHERSUITE</name>
|
||||||
|
<value></value>
|
||||||
|
<description>
|
||||||
|
The prefix for a given crypto codec, contains a comma-separated
|
||||||
|
list of implementation classes for a given crypto codec (eg EXAMPLECIPHERSUITE).
|
||||||
|
The first implementation will be used if available, others are fallbacks.
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>hadoop.security.crypto.codec.classes.aes.ctr.nopadding</name>
|
||||||
|
<value>org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec,org.apache.hadoop.crypto.JceAesCtrCryptoCodec</value>
|
||||||
|
<description>
|
||||||
|
Comma-separated list of crypto codec implementations for AES/CTR/NoPadding.
|
||||||
|
The first implementation will be used if available, others are fallbacks.
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>hadoop.security.crypto.cipher.suite</name>
|
||||||
|
<value>AES/CTR/NoPadding</value>
|
||||||
|
<description>
|
||||||
|
Cipher suite for crypto codec.
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>hadoop.security.crypto.jce.provider</name>
|
||||||
|
<value></value>
|
||||||
|
<description>
|
||||||
|
The JCE provider name used in CryptoCodec.
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>hadoop.security.crypto.buffer.size</name>
|
||||||
|
<value>8192</value>
|
||||||
|
<description>
|
||||||
|
The buffer size used by CryptoInputStream and CryptoOutputStream.
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>hadoop.security.java.secure.random.algorithm</name>
|
||||||
|
<value>SHA1PRNG</value>
|
||||||
|
<description>
|
||||||
|
The java secure random algorithm.
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>hadoop.security.secure.random.impl</name>
|
||||||
|
<value></value>
|
||||||
|
<description>
|
||||||
|
Implementation of secure random.
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>hadoop.security.random.device.file.path</name>
|
||||||
|
<value>/dev/urandom</value>
|
||||||
|
<description>
|
||||||
|
OS security random device file path.
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
<property>
|
<property>
|
||||||
<name>fs.har.impl.disable.cache</name>
|
<name>fs.har.impl.disable.cache</name>
|
||||||
<value>true</value>
|
<value>true</value>
|
||||||
|
@ -1483,4 +1551,5 @@ for ldap providers in the same way as above does.
|
||||||
key will be dropped. Default = 12hrs
|
key will be dropped. Default = 12hrs
|
||||||
</description>
|
</description>
|
||||||
</property>
|
</property>
|
||||||
|
|
||||||
</configuration>
|
</configuration>
|
||||||
|
|
|
@ -168,15 +168,22 @@ cp
|
||||||
Copy files from source to destination. This command allows multiple sources
|
Copy files from source to destination. This command allows multiple sources
|
||||||
as well in which case the destination must be a directory.
|
as well in which case the destination must be a directory.
|
||||||
|
|
||||||
|
'raw.*' namespace extended attributes are preserved if (1) the source and
|
||||||
|
destination filesystems support them (HDFS only), and (2) all source and
|
||||||
|
destination pathnames are in the /.reserved/raw hierarchy. Determination of
|
||||||
|
whether raw.* namespace xattrs are preserved is independent of the
|
||||||
|
-p (preserve) flag.
|
||||||
|
|
||||||
Options:
|
Options:
|
||||||
|
|
||||||
* The -f option will overwrite the destination if it already exists.
|
* The -f option will overwrite the destination if it already exists.
|
||||||
|
|
||||||
* The -p option will preserve file attributes [topx] (timestamps,
|
* The -p option will preserve file attributes [topx] (timestamps,
|
||||||
ownership, permission, ACL, XAttr). If -p is specified with no <arg>,
|
ownership, permission, ACL, XAttr). If -p is specified with no <arg>,
|
||||||
then preserves timestamps, ownership, permission. If -pa is specified,
|
then preserves timestamps, ownership, permission. If -pa is specified,
|
||||||
then preserves permission also because ACL is a super-set of
|
then preserves permission also because ACL is a super-set of
|
||||||
permission.
|
permission. Determination of whether raw namespace extended attributes
|
||||||
|
are preserved is independent of the -p flag.
|
||||||
|
|
||||||
Example:
|
Example:
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,721 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.crypto;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.io.OutputStream;
|
||||||
|
import java.nio.ByteBuffer;
|
||||||
|
import java.nio.ByteOrder;
|
||||||
|
import java.util.EnumSet;
|
||||||
|
import java.util.Random;
|
||||||
|
|
||||||
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
import org.apache.hadoop.fs.ByteBufferReadable;
|
||||||
|
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||||
|
import org.apache.hadoop.fs.HasEnhancedByteBufferAccess;
|
||||||
|
import org.apache.hadoop.fs.PositionedReadable;
|
||||||
|
import org.apache.hadoop.fs.ReadOption;
|
||||||
|
import org.apache.hadoop.fs.Seekable;
|
||||||
|
import org.apache.hadoop.fs.Syncable;
|
||||||
|
import org.apache.hadoop.io.ByteBufferPool;
|
||||||
|
import org.apache.hadoop.io.DataOutputBuffer;
|
||||||
|
import org.apache.hadoop.io.RandomDatum;
|
||||||
|
import org.apache.hadoop.test.GenericTestUtils;
|
||||||
|
import org.junit.Assert;
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
public abstract class CryptoStreamsTestBase {
|
||||||
|
protected static final Log LOG = LogFactory.getLog(
|
||||||
|
CryptoStreamsTestBase.class);
|
||||||
|
|
||||||
|
protected static CryptoCodec codec;
|
||||||
|
private static final byte[] key = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
|
||||||
|
0x07, 0x08, 0x09, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16};
|
||||||
|
private static final byte[] iv = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
|
||||||
|
0x07, 0x08, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08};
|
||||||
|
|
||||||
|
protected static final int count = 10000;
|
||||||
|
protected static int defaultBufferSize = 8192;
|
||||||
|
protected static int smallBufferSize = 1024;
|
||||||
|
private byte[] data;
|
||||||
|
private int dataLen;
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void setUp() throws IOException {
|
||||||
|
// Generate data
|
||||||
|
final int seed = new Random().nextInt();
|
||||||
|
final DataOutputBuffer dataBuf = new DataOutputBuffer();
|
||||||
|
final RandomDatum.Generator generator = new RandomDatum.Generator(seed);
|
||||||
|
for(int i = 0; i < count; ++i) {
|
||||||
|
generator.next();
|
||||||
|
final RandomDatum key = generator.getKey();
|
||||||
|
final RandomDatum value = generator.getValue();
|
||||||
|
|
||||||
|
key.write(dataBuf);
|
||||||
|
value.write(dataBuf);
|
||||||
|
}
|
||||||
|
LOG.info("Generated " + count + " records");
|
||||||
|
data = dataBuf.getData();
|
||||||
|
dataLen = dataBuf.getLength();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void writeData(OutputStream out) throws Exception {
|
||||||
|
out.write(data, 0, dataLen);
|
||||||
|
out.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected int getDataLen() {
|
||||||
|
return dataLen;
|
||||||
|
}
|
||||||
|
|
||||||
|
private int readAll(InputStream in, byte[] b, int off, int len)
|
||||||
|
throws IOException {
|
||||||
|
int n = 0;
|
||||||
|
int total = 0;
|
||||||
|
while (n != -1) {
|
||||||
|
total += n;
|
||||||
|
if (total >= len) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
n = in.read(b, off + total, len - total);
|
||||||
|
}
|
||||||
|
|
||||||
|
return total;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected OutputStream getOutputStream(int bufferSize) throws IOException {
|
||||||
|
return getOutputStream(bufferSize, key, iv);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected abstract OutputStream getOutputStream(int bufferSize, byte[] key,
|
||||||
|
byte[] iv) throws IOException;
|
||||||
|
|
||||||
|
protected InputStream getInputStream(int bufferSize) throws IOException {
|
||||||
|
return getInputStream(bufferSize, key, iv);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected abstract InputStream getInputStream(int bufferSize, byte[] key,
|
||||||
|
byte[] iv) throws IOException;
|
||||||
|
|
||||||
|
/** Test crypto reading with different buffer size. */
|
||||||
|
@Test(timeout=120000)
|
||||||
|
public void testRead() throws Exception {
|
||||||
|
OutputStream out = getOutputStream(defaultBufferSize);
|
||||||
|
writeData(out);
|
||||||
|
|
||||||
|
// Default buffer size
|
||||||
|
InputStream in = getInputStream(defaultBufferSize);
|
||||||
|
readCheck(in);
|
||||||
|
in.close();
|
||||||
|
|
||||||
|
// Small buffer size
|
||||||
|
in = getInputStream(smallBufferSize);
|
||||||
|
readCheck(in);
|
||||||
|
in.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void readCheck(InputStream in) throws Exception {
|
||||||
|
byte[] result = new byte[dataLen];
|
||||||
|
int n = readAll(in, result, 0, dataLen);
|
||||||
|
|
||||||
|
Assert.assertEquals(dataLen, n);
|
||||||
|
byte[] expectedData = new byte[n];
|
||||||
|
System.arraycopy(data, 0, expectedData, 0, n);
|
||||||
|
Assert.assertArrayEquals(result, expectedData);
|
||||||
|
|
||||||
|
// EOF
|
||||||
|
n = in.read(result, 0, dataLen);
|
||||||
|
Assert.assertEquals(n, -1);
|
||||||
|
in.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Test crypto writing with different buffer size. */
|
||||||
|
@Test(timeout = 120000)
|
||||||
|
public void testWrite() throws Exception {
|
||||||
|
// Default buffer size
|
||||||
|
writeCheck(defaultBufferSize);
|
||||||
|
|
||||||
|
// Small buffer size
|
||||||
|
writeCheck(smallBufferSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void writeCheck(int bufferSize) throws Exception {
|
||||||
|
OutputStream out = getOutputStream(bufferSize);
|
||||||
|
writeData(out);
|
||||||
|
|
||||||
|
if (out instanceof FSDataOutputStream) {
|
||||||
|
Assert.assertEquals(((FSDataOutputStream) out).getPos(), getDataLen());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Test crypto with different IV. */
|
||||||
|
@Test(timeout=120000)
|
||||||
|
public void testCryptoIV() throws Exception {
|
||||||
|
byte[] iv1 = iv.clone();
|
||||||
|
|
||||||
|
// Counter base: Long.MAX_VALUE
|
||||||
|
setCounterBaseForIV(iv1, Long.MAX_VALUE);
|
||||||
|
cryptoCheck(iv1);
|
||||||
|
|
||||||
|
// Counter base: Long.MAX_VALUE - 1
|
||||||
|
setCounterBaseForIV(iv1, Long.MAX_VALUE - 1);
|
||||||
|
cryptoCheck(iv1);
|
||||||
|
|
||||||
|
// Counter base: Integer.MAX_VALUE
|
||||||
|
setCounterBaseForIV(iv1, Integer.MAX_VALUE);
|
||||||
|
cryptoCheck(iv1);
|
||||||
|
|
||||||
|
// Counter base: 0
|
||||||
|
setCounterBaseForIV(iv1, 0);
|
||||||
|
cryptoCheck(iv1);
|
||||||
|
|
||||||
|
// Counter base: -1
|
||||||
|
setCounterBaseForIV(iv1, -1);
|
||||||
|
cryptoCheck(iv1);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void cryptoCheck(byte[] iv) throws Exception {
|
||||||
|
OutputStream out = getOutputStream(defaultBufferSize, key, iv);
|
||||||
|
writeData(out);
|
||||||
|
|
||||||
|
InputStream in = getInputStream(defaultBufferSize, key, iv);
|
||||||
|
readCheck(in);
|
||||||
|
in.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void setCounterBaseForIV(byte[] iv, long counterBase) {
|
||||||
|
ByteBuffer buf = ByteBuffer.wrap(iv);
|
||||||
|
buf.order(ByteOrder.BIG_ENDIAN);
|
||||||
|
buf.putLong(iv.length - 8, counterBase);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test hflush/hsync of crypto output stream, and with different buffer size.
|
||||||
|
*/
|
||||||
|
@Test(timeout=120000)
|
||||||
|
public void testSyncable() throws IOException {
|
||||||
|
syncableCheck();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void syncableCheck() throws IOException {
|
||||||
|
OutputStream out = getOutputStream(smallBufferSize);
|
||||||
|
try {
|
||||||
|
int bytesWritten = dataLen / 3;
|
||||||
|
out.write(data, 0, bytesWritten);
|
||||||
|
((Syncable) out).hflush();
|
||||||
|
|
||||||
|
InputStream in = getInputStream(defaultBufferSize);
|
||||||
|
verify(in, bytesWritten, data);
|
||||||
|
in.close();
|
||||||
|
|
||||||
|
out.write(data, bytesWritten, dataLen - bytesWritten);
|
||||||
|
((Syncable) out).hsync();
|
||||||
|
|
||||||
|
in = getInputStream(defaultBufferSize);
|
||||||
|
verify(in, dataLen, data);
|
||||||
|
in.close();
|
||||||
|
} finally {
|
||||||
|
out.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void verify(InputStream in, int bytesToVerify,
|
||||||
|
byte[] expectedBytes) throws IOException {
|
||||||
|
final byte[] readBuf = new byte[bytesToVerify];
|
||||||
|
readAll(in, readBuf, 0, bytesToVerify);
|
||||||
|
for (int i = 0; i < bytesToVerify; i++) {
|
||||||
|
Assert.assertEquals(expectedBytes[i], readBuf[i]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private int readAll(InputStream in, long pos, byte[] b, int off, int len)
|
||||||
|
throws IOException {
|
||||||
|
int n = 0;
|
||||||
|
int total = 0;
|
||||||
|
while (n != -1) {
|
||||||
|
total += n;
|
||||||
|
if (total >= len) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
n = ((PositionedReadable) in).read(pos + total, b, off + total,
|
||||||
|
len - total);
|
||||||
|
}
|
||||||
|
|
||||||
|
return total;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Test positioned read. */
|
||||||
|
@Test(timeout=120000)
|
||||||
|
public void testPositionedRead() throws Exception {
|
||||||
|
OutputStream out = getOutputStream(defaultBufferSize);
|
||||||
|
writeData(out);
|
||||||
|
|
||||||
|
InputStream in = getInputStream(defaultBufferSize);
|
||||||
|
// Pos: 1/3 dataLen
|
||||||
|
positionedReadCheck(in , dataLen / 3);
|
||||||
|
|
||||||
|
// Pos: 1/2 dataLen
|
||||||
|
positionedReadCheck(in, dataLen / 2);
|
||||||
|
in.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void positionedReadCheck(InputStream in, int pos) throws Exception {
|
||||||
|
byte[] result = new byte[dataLen];
|
||||||
|
int n = readAll(in, pos, result, 0, dataLen);
|
||||||
|
|
||||||
|
Assert.assertEquals(dataLen, n + pos);
|
||||||
|
byte[] readData = new byte[n];
|
||||||
|
System.arraycopy(result, 0, readData, 0, n);
|
||||||
|
byte[] expectedData = new byte[n];
|
||||||
|
System.arraycopy(data, pos, expectedData, 0, n);
|
||||||
|
Assert.assertArrayEquals(readData, expectedData);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Test read fully */
|
||||||
|
@Test(timeout=120000)
|
||||||
|
public void testReadFully() throws Exception {
|
||||||
|
OutputStream out = getOutputStream(defaultBufferSize);
|
||||||
|
writeData(out);
|
||||||
|
|
||||||
|
InputStream in = getInputStream(defaultBufferSize);
|
||||||
|
final int len1 = dataLen / 4;
|
||||||
|
// Read len1 bytes
|
||||||
|
byte[] readData = new byte[len1];
|
||||||
|
readAll(in, readData, 0, len1);
|
||||||
|
byte[] expectedData = new byte[len1];
|
||||||
|
System.arraycopy(data, 0, expectedData, 0, len1);
|
||||||
|
Assert.assertArrayEquals(readData, expectedData);
|
||||||
|
|
||||||
|
// Pos: 1/3 dataLen
|
||||||
|
readFullyCheck(in, dataLen / 3);
|
||||||
|
|
||||||
|
// Read len1 bytes
|
||||||
|
readData = new byte[len1];
|
||||||
|
readAll(in, readData, 0, len1);
|
||||||
|
expectedData = new byte[len1];
|
||||||
|
System.arraycopy(data, len1, expectedData, 0, len1);
|
||||||
|
Assert.assertArrayEquals(readData, expectedData);
|
||||||
|
|
||||||
|
// Pos: 1/2 dataLen
|
||||||
|
readFullyCheck(in, dataLen / 2);
|
||||||
|
|
||||||
|
// Read len1 bytes
|
||||||
|
readData = new byte[len1];
|
||||||
|
readAll(in, readData, 0, len1);
|
||||||
|
expectedData = new byte[len1];
|
||||||
|
System.arraycopy(data, 2 * len1, expectedData, 0, len1);
|
||||||
|
Assert.assertArrayEquals(readData, expectedData);
|
||||||
|
|
||||||
|
in.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void readFullyCheck(InputStream in, int pos) throws Exception {
|
||||||
|
byte[] result = new byte[dataLen - pos];
|
||||||
|
((PositionedReadable) in).readFully(pos, result);
|
||||||
|
|
||||||
|
byte[] expectedData = new byte[dataLen - pos];
|
||||||
|
System.arraycopy(data, pos, expectedData, 0, dataLen - pos);
|
||||||
|
Assert.assertArrayEquals(result, expectedData);
|
||||||
|
|
||||||
|
result = new byte[dataLen]; // Exceeds maximum length
|
||||||
|
try {
|
||||||
|
((PositionedReadable) in).readFully(pos, result);
|
||||||
|
Assert.fail("Read fully exceeds maximum length should fail.");
|
||||||
|
} catch (IOException e) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Test seek to different position. */
|
||||||
|
@Test(timeout=120000)
|
||||||
|
public void testSeek() throws Exception {
|
||||||
|
OutputStream out = getOutputStream(defaultBufferSize);
|
||||||
|
writeData(out);
|
||||||
|
|
||||||
|
InputStream in = getInputStream(defaultBufferSize);
|
||||||
|
// Pos: 1/3 dataLen
|
||||||
|
seekCheck(in, dataLen / 3);
|
||||||
|
|
||||||
|
// Pos: 0
|
||||||
|
seekCheck(in, 0);
|
||||||
|
|
||||||
|
// Pos: 1/2 dataLen
|
||||||
|
seekCheck(in, dataLen / 2);
|
||||||
|
|
||||||
|
final long pos = ((Seekable) in).getPos();
|
||||||
|
|
||||||
|
// Pos: -3
|
||||||
|
try {
|
||||||
|
seekCheck(in, -3);
|
||||||
|
Assert.fail("Seek to negative offset should fail.");
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
GenericTestUtils.assertExceptionContains("Cannot seek to negative " +
|
||||||
|
"offset", e);
|
||||||
|
}
|
||||||
|
Assert.assertEquals(pos, ((Seekable) in).getPos());
|
||||||
|
|
||||||
|
// Pos: dataLen + 3
|
||||||
|
try {
|
||||||
|
seekCheck(in, dataLen + 3);
|
||||||
|
Assert.fail("Seek after EOF should fail.");
|
||||||
|
} catch (IOException e) {
|
||||||
|
GenericTestUtils.assertExceptionContains("Cannot seek after EOF", e);
|
||||||
|
}
|
||||||
|
Assert.assertEquals(pos, ((Seekable) in).getPos());
|
||||||
|
|
||||||
|
in.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void seekCheck(InputStream in, int pos) throws Exception {
|
||||||
|
byte[] result = new byte[dataLen];
|
||||||
|
((Seekable) in).seek(pos);
|
||||||
|
int n = readAll(in, result, 0, dataLen);
|
||||||
|
|
||||||
|
Assert.assertEquals(dataLen, n + pos);
|
||||||
|
byte[] readData = new byte[n];
|
||||||
|
System.arraycopy(result, 0, readData, 0, n);
|
||||||
|
byte[] expectedData = new byte[n];
|
||||||
|
System.arraycopy(data, pos, expectedData, 0, n);
|
||||||
|
Assert.assertArrayEquals(readData, expectedData);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Test get position. */
|
||||||
|
@Test(timeout=120000)
|
||||||
|
public void testGetPos() throws Exception {
|
||||||
|
OutputStream out = getOutputStream(defaultBufferSize);
|
||||||
|
writeData(out);
|
||||||
|
|
||||||
|
// Default buffer size
|
||||||
|
InputStream in = getInputStream(defaultBufferSize);
|
||||||
|
byte[] result = new byte[dataLen];
|
||||||
|
int n1 = readAll(in, result, 0, dataLen / 3);
|
||||||
|
Assert.assertEquals(n1, ((Seekable) in).getPos());
|
||||||
|
|
||||||
|
int n2 = readAll(in, result, n1, dataLen - n1);
|
||||||
|
Assert.assertEquals(n1 + n2, ((Seekable) in).getPos());
|
||||||
|
in.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout=120000)
|
||||||
|
public void testAvailable() throws Exception {
|
||||||
|
OutputStream out = getOutputStream(defaultBufferSize);
|
||||||
|
writeData(out);
|
||||||
|
|
||||||
|
// Default buffer size
|
||||||
|
InputStream in = getInputStream(defaultBufferSize);
|
||||||
|
byte[] result = new byte[dataLen];
|
||||||
|
int n1 = readAll(in, result, 0, dataLen / 3);
|
||||||
|
Assert.assertEquals(in.available(), dataLen - n1);
|
||||||
|
|
||||||
|
int n2 = readAll(in, result, n1, dataLen - n1);
|
||||||
|
Assert.assertEquals(in.available(), dataLen - n1 - n2);
|
||||||
|
in.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Test skip. */
|
||||||
|
@Test(timeout=120000)
|
||||||
|
public void testSkip() throws Exception {
|
||||||
|
OutputStream out = getOutputStream(defaultBufferSize);
|
||||||
|
writeData(out);
|
||||||
|
|
||||||
|
// Default buffer size
|
||||||
|
InputStream in = getInputStream(defaultBufferSize);
|
||||||
|
byte[] result = new byte[dataLen];
|
||||||
|
int n1 = readAll(in, result, 0, dataLen / 3);
|
||||||
|
Assert.assertEquals(n1, ((Seekable) in).getPos());
|
||||||
|
|
||||||
|
long skipped = in.skip(dataLen / 3);
|
||||||
|
int n2 = readAll(in, result, 0, dataLen);
|
||||||
|
|
||||||
|
Assert.assertEquals(dataLen, n1 + skipped + n2);
|
||||||
|
byte[] readData = new byte[n2];
|
||||||
|
System.arraycopy(result, 0, readData, 0, n2);
|
||||||
|
byte[] expectedData = new byte[n2];
|
||||||
|
System.arraycopy(data, dataLen - n2, expectedData, 0, n2);
|
||||||
|
Assert.assertArrayEquals(readData, expectedData);
|
||||||
|
|
||||||
|
try {
|
||||||
|
skipped = in.skip(-3);
|
||||||
|
Assert.fail("Skip Negative length should fail.");
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
GenericTestUtils.assertExceptionContains("Negative skip length", e);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Skip after EOF
|
||||||
|
skipped = in.skip(3);
|
||||||
|
Assert.assertEquals(skipped, 0);
|
||||||
|
|
||||||
|
in.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void byteBufferReadCheck(InputStream in, ByteBuffer buf,
|
||||||
|
int bufPos) throws Exception {
|
||||||
|
buf.position(bufPos);
|
||||||
|
int n = ((ByteBufferReadable) in).read(buf);
|
||||||
|
byte[] readData = new byte[n];
|
||||||
|
buf.rewind();
|
||||||
|
buf.position(bufPos);
|
||||||
|
buf.get(readData);
|
||||||
|
byte[] expectedData = new byte[n];
|
||||||
|
System.arraycopy(data, 0, expectedData, 0, n);
|
||||||
|
Assert.assertArrayEquals(readData, expectedData);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Test byte buffer read with different buffer size. */
|
||||||
|
@Test(timeout=120000)
|
||||||
|
public void testByteBufferRead() throws Exception {
|
||||||
|
OutputStream out = getOutputStream(defaultBufferSize);
|
||||||
|
writeData(out);
|
||||||
|
|
||||||
|
// Default buffer size, initial buffer position is 0
|
||||||
|
InputStream in = getInputStream(defaultBufferSize);
|
||||||
|
ByteBuffer buf = ByteBuffer.allocate(dataLen + 100);
|
||||||
|
byteBufferReadCheck(in, buf, 0);
|
||||||
|
in.close();
|
||||||
|
|
||||||
|
// Default buffer size, initial buffer position is not 0
|
||||||
|
in = getInputStream(defaultBufferSize);
|
||||||
|
buf.clear();
|
||||||
|
byteBufferReadCheck(in, buf, 11);
|
||||||
|
in.close();
|
||||||
|
|
||||||
|
// Small buffer size, initial buffer position is 0
|
||||||
|
in = getInputStream(smallBufferSize);
|
||||||
|
buf.clear();
|
||||||
|
byteBufferReadCheck(in, buf, 0);
|
||||||
|
in.close();
|
||||||
|
|
||||||
|
// Small buffer size, initial buffer position is not 0
|
||||||
|
in = getInputStream(smallBufferSize);
|
||||||
|
buf.clear();
|
||||||
|
byteBufferReadCheck(in, buf, 11);
|
||||||
|
in.close();
|
||||||
|
|
||||||
|
// Direct buffer, default buffer size, initial buffer position is 0
|
||||||
|
in = getInputStream(defaultBufferSize);
|
||||||
|
buf = ByteBuffer.allocateDirect(dataLen + 100);
|
||||||
|
byteBufferReadCheck(in, buf, 0);
|
||||||
|
in.close();
|
||||||
|
|
||||||
|
// Direct buffer, default buffer size, initial buffer position is not 0
|
||||||
|
in = getInputStream(defaultBufferSize);
|
||||||
|
buf.clear();
|
||||||
|
byteBufferReadCheck(in, buf, 11);
|
||||||
|
in.close();
|
||||||
|
|
||||||
|
// Direct buffer, small buffer size, initial buffer position is 0
|
||||||
|
in = getInputStream(smallBufferSize);
|
||||||
|
buf.clear();
|
||||||
|
byteBufferReadCheck(in, buf, 0);
|
||||||
|
in.close();
|
||||||
|
|
||||||
|
// Direct buffer, small buffer size, initial buffer position is not 0
|
||||||
|
in = getInputStream(smallBufferSize);
|
||||||
|
buf.clear();
|
||||||
|
byteBufferReadCheck(in, buf, 11);
|
||||||
|
in.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout=120000)
|
||||||
|
public void testCombinedOp() throws Exception {
|
||||||
|
OutputStream out = getOutputStream(defaultBufferSize);
|
||||||
|
writeData(out);
|
||||||
|
|
||||||
|
final int len1 = dataLen / 8;
|
||||||
|
final int len2 = dataLen / 10;
|
||||||
|
|
||||||
|
InputStream in = getInputStream(defaultBufferSize);
|
||||||
|
// Read len1 data.
|
||||||
|
byte[] readData = new byte[len1];
|
||||||
|
readAll(in, readData, 0, len1);
|
||||||
|
byte[] expectedData = new byte[len1];
|
||||||
|
System.arraycopy(data, 0, expectedData, 0, len1);
|
||||||
|
Assert.assertArrayEquals(readData, expectedData);
|
||||||
|
|
||||||
|
long pos = ((Seekable) in).getPos();
|
||||||
|
Assert.assertEquals(len1, pos);
|
||||||
|
|
||||||
|
// Seek forward len2
|
||||||
|
((Seekable) in).seek(pos + len2);
|
||||||
|
// Skip forward len2
|
||||||
|
long n = in.skip(len2);
|
||||||
|
Assert.assertEquals(len2, n);
|
||||||
|
|
||||||
|
// Pos: 1/4 dataLen
|
||||||
|
positionedReadCheck(in , dataLen / 4);
|
||||||
|
|
||||||
|
// Pos should be len1 + len2 + len2
|
||||||
|
pos = ((Seekable) in).getPos();
|
||||||
|
Assert.assertEquals(len1 + len2 + len2, pos);
|
||||||
|
|
||||||
|
// Read forward len1
|
||||||
|
ByteBuffer buf = ByteBuffer.allocate(len1);
|
||||||
|
int nRead = ((ByteBufferReadable) in).read(buf);
|
||||||
|
readData = new byte[nRead];
|
||||||
|
buf.rewind();
|
||||||
|
buf.get(readData);
|
||||||
|
expectedData = new byte[nRead];
|
||||||
|
System.arraycopy(data, (int)pos, expectedData, 0, nRead);
|
||||||
|
Assert.assertArrayEquals(readData, expectedData);
|
||||||
|
|
||||||
|
// Pos should be len1 + 2 * len2 + nRead
|
||||||
|
pos = ((Seekable) in).getPos();
|
||||||
|
Assert.assertEquals(len1 + 2 * len2 + nRead, pos);
|
||||||
|
|
||||||
|
// Pos: 1/3 dataLen
|
||||||
|
positionedReadCheck(in , dataLen / 3);
|
||||||
|
|
||||||
|
// Read forward len1
|
||||||
|
readData = new byte[len1];
|
||||||
|
readAll(in, readData, 0, len1);
|
||||||
|
expectedData = new byte[len1];
|
||||||
|
System.arraycopy(data, (int)pos, expectedData, 0, len1);
|
||||||
|
Assert.assertArrayEquals(readData, expectedData);
|
||||||
|
|
||||||
|
// Pos should be 2 * len1 + 2 * len2 + nRead
|
||||||
|
pos = ((Seekable) in).getPos();
|
||||||
|
Assert.assertEquals(2 * len1 + 2 * len2 + nRead, pos);
|
||||||
|
|
||||||
|
// Read forward len1
|
||||||
|
buf = ByteBuffer.allocate(len1);
|
||||||
|
nRead = ((ByteBufferReadable) in).read(buf);
|
||||||
|
readData = new byte[nRead];
|
||||||
|
buf.rewind();
|
||||||
|
buf.get(readData);
|
||||||
|
expectedData = new byte[nRead];
|
||||||
|
System.arraycopy(data, (int)pos, expectedData, 0, nRead);
|
||||||
|
Assert.assertArrayEquals(readData, expectedData);
|
||||||
|
|
||||||
|
// ByteBuffer read after EOF
|
||||||
|
((Seekable) in).seek(dataLen);
|
||||||
|
buf.clear();
|
||||||
|
n = ((ByteBufferReadable) in).read(buf);
|
||||||
|
Assert.assertEquals(n, -1);
|
||||||
|
|
||||||
|
in.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout=120000)
|
||||||
|
public void testSeekToNewSource() throws Exception {
|
||||||
|
OutputStream out = getOutputStream(defaultBufferSize);
|
||||||
|
writeData(out);
|
||||||
|
|
||||||
|
InputStream in = getInputStream(defaultBufferSize);
|
||||||
|
|
||||||
|
final int len1 = dataLen / 8;
|
||||||
|
byte[] readData = new byte[len1];
|
||||||
|
readAll(in, readData, 0, len1);
|
||||||
|
|
||||||
|
// Pos: 1/3 dataLen
|
||||||
|
seekToNewSourceCheck(in, dataLen / 3);
|
||||||
|
|
||||||
|
// Pos: 0
|
||||||
|
seekToNewSourceCheck(in, 0);
|
||||||
|
|
||||||
|
// Pos: 1/2 dataLen
|
||||||
|
seekToNewSourceCheck(in, dataLen / 2);
|
||||||
|
|
||||||
|
// Pos: -3
|
||||||
|
try {
|
||||||
|
seekToNewSourceCheck(in, -3);
|
||||||
|
Assert.fail("Seek to negative offset should fail.");
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
GenericTestUtils.assertExceptionContains("Cannot seek to negative " +
|
||||||
|
"offset", e);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pos: dataLen + 3
|
||||||
|
try {
|
||||||
|
seekToNewSourceCheck(in, dataLen + 3);
|
||||||
|
Assert.fail("Seek after EOF should fail.");
|
||||||
|
} catch (IOException e) {
|
||||||
|
GenericTestUtils.assertExceptionContains("Attempted to read past " +
|
||||||
|
"end of file", e);
|
||||||
|
}
|
||||||
|
|
||||||
|
in.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void seekToNewSourceCheck(InputStream in, int targetPos)
|
||||||
|
throws Exception {
|
||||||
|
byte[] result = new byte[dataLen];
|
||||||
|
((Seekable) in).seekToNewSource(targetPos);
|
||||||
|
int n = readAll(in, result, 0, dataLen);
|
||||||
|
|
||||||
|
Assert.assertEquals(dataLen, n + targetPos);
|
||||||
|
byte[] readData = new byte[n];
|
||||||
|
System.arraycopy(result, 0, readData, 0, n);
|
||||||
|
byte[] expectedData = new byte[n];
|
||||||
|
System.arraycopy(data, targetPos, expectedData, 0, n);
|
||||||
|
Assert.assertArrayEquals(readData, expectedData);
|
||||||
|
}
|
||||||
|
|
||||||
|
private ByteBufferPool getBufferPool() {
|
||||||
|
return new ByteBufferPool() {
|
||||||
|
@Override
|
||||||
|
public ByteBuffer getBuffer(boolean direct, int length) {
|
||||||
|
return ByteBuffer.allocateDirect(length);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void putBuffer(ByteBuffer buffer) {
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout=120000)
|
||||||
|
public void testHasEnhancedByteBufferAccess() throws Exception {
|
||||||
|
OutputStream out = getOutputStream(defaultBufferSize);
|
||||||
|
writeData(out);
|
||||||
|
|
||||||
|
InputStream in = getInputStream(defaultBufferSize);
|
||||||
|
final int len1 = dataLen / 8;
|
||||||
|
// ByteBuffer size is len1
|
||||||
|
ByteBuffer buffer = ((HasEnhancedByteBufferAccess) in).read(
|
||||||
|
getBufferPool(), len1, EnumSet.of(ReadOption.SKIP_CHECKSUMS));
|
||||||
|
int n1 = buffer.remaining();
|
||||||
|
byte[] readData = new byte[n1];
|
||||||
|
buffer.get(readData);
|
||||||
|
byte[] expectedData = new byte[n1];
|
||||||
|
System.arraycopy(data, 0, expectedData, 0, n1);
|
||||||
|
Assert.assertArrayEquals(readData, expectedData);
|
||||||
|
((HasEnhancedByteBufferAccess) in).releaseBuffer(buffer);
|
||||||
|
|
||||||
|
// Read len1 bytes
|
||||||
|
readData = new byte[len1];
|
||||||
|
readAll(in, readData, 0, len1);
|
||||||
|
expectedData = new byte[len1];
|
||||||
|
System.arraycopy(data, n1, expectedData, 0, len1);
|
||||||
|
Assert.assertArrayEquals(readData, expectedData);
|
||||||
|
|
||||||
|
// ByteBuffer size is len1
|
||||||
|
buffer = ((HasEnhancedByteBufferAccess) in).read(
|
||||||
|
getBufferPool(), len1, EnumSet.of(ReadOption.SKIP_CHECKSUMS));
|
||||||
|
int n2 = buffer.remaining();
|
||||||
|
readData = new byte[n2];
|
||||||
|
buffer.get(readData);
|
||||||
|
expectedData = new byte[n2];
|
||||||
|
System.arraycopy(data, n1 + len1, expectedData, 0, n2);
|
||||||
|
Assert.assertArrayEquals(readData, expectedData);
|
||||||
|
((HasEnhancedByteBufferAccess) in).releaseBuffer(buffer);
|
||||||
|
|
||||||
|
in.close();
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,186 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.crypto;
|
||||||
|
|
||||||
|
import static org.junit.Assert.assertEquals;
|
||||||
|
import static org.junit.Assert.assertTrue;
|
||||||
|
|
||||||
|
import java.io.BufferedInputStream;
|
||||||
|
import java.io.DataInputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.security.GeneralSecurityException;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Random;
|
||||||
|
|
||||||
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.io.DataInputBuffer;
|
||||||
|
import org.apache.hadoop.io.DataOutputBuffer;
|
||||||
|
import org.apache.hadoop.io.RandomDatum;
|
||||||
|
import org.apache.hadoop.util.NativeCodeLoader;
|
||||||
|
import org.apache.hadoop.util.ReflectionUtils;
|
||||||
|
import org.junit.Assert;
|
||||||
|
import org.junit.Assume;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
public class TestCryptoCodec {
|
||||||
|
private static final Log LOG= LogFactory.getLog(TestCryptoCodec.class);
|
||||||
|
private static final byte[] key = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
|
||||||
|
0x07, 0x08, 0x09, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16};
|
||||||
|
private static final byte[] iv = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
|
||||||
|
0x07, 0x08, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08};
|
||||||
|
private static final int bufferSize = 4096;
|
||||||
|
|
||||||
|
private Configuration conf = new Configuration();
|
||||||
|
private int count = 10000;
|
||||||
|
private int seed = new Random().nextInt();
|
||||||
|
|
||||||
|
@Test(timeout=120000)
|
||||||
|
public void testJceAesCtrCryptoCodec() throws Exception {
|
||||||
|
cryptoCodecTest(conf, seed, 0,
|
||||||
|
"org.apache.hadoop.crypto.JceAesCtrCryptoCodec");
|
||||||
|
cryptoCodecTest(conf, seed, count,
|
||||||
|
"org.apache.hadoop.crypto.JceAesCtrCryptoCodec");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout=1200000)
|
||||||
|
public void testOpensslAesCtrCryptoCodec() throws Exception {
|
||||||
|
Assume.assumeTrue(NativeCodeLoader.buildSupportsOpenssl());
|
||||||
|
Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason());
|
||||||
|
cryptoCodecTest(conf, seed, 0,
|
||||||
|
"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec");
|
||||||
|
cryptoCodecTest(conf, seed, count,
|
||||||
|
"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec");
|
||||||
|
}
|
||||||
|
|
||||||
|
private void cryptoCodecTest(Configuration conf, int seed, int count,
|
||||||
|
String codecClass) throws IOException, GeneralSecurityException {
|
||||||
|
CryptoCodec codec = null;
|
||||||
|
try {
|
||||||
|
codec = (CryptoCodec)ReflectionUtils.newInstance(
|
||||||
|
conf.getClassByName(codecClass), conf);
|
||||||
|
} catch (ClassNotFoundException cnfe) {
|
||||||
|
throw new IOException("Illegal crypto codec!");
|
||||||
|
}
|
||||||
|
LOG.info("Created a Codec object of type: " + codecClass);
|
||||||
|
|
||||||
|
// Generate data
|
||||||
|
DataOutputBuffer data = new DataOutputBuffer();
|
||||||
|
RandomDatum.Generator generator = new RandomDatum.Generator(seed);
|
||||||
|
for(int i = 0; i < count; ++i) {
|
||||||
|
generator.next();
|
||||||
|
RandomDatum key = generator.getKey();
|
||||||
|
RandomDatum value = generator.getValue();
|
||||||
|
|
||||||
|
key.write(data);
|
||||||
|
value.write(data);
|
||||||
|
}
|
||||||
|
LOG.info("Generated " + count + " records");
|
||||||
|
|
||||||
|
// Encrypt data
|
||||||
|
DataOutputBuffer encryptedDataBuffer = new DataOutputBuffer();
|
||||||
|
CryptoOutputStream out = new CryptoOutputStream(encryptedDataBuffer,
|
||||||
|
codec, bufferSize, key, iv);
|
||||||
|
out.write(data.getData(), 0, data.getLength());
|
||||||
|
out.flush();
|
||||||
|
out.close();
|
||||||
|
LOG.info("Finished encrypting data");
|
||||||
|
|
||||||
|
// Decrypt data
|
||||||
|
DataInputBuffer decryptedDataBuffer = new DataInputBuffer();
|
||||||
|
decryptedDataBuffer.reset(encryptedDataBuffer.getData(), 0,
|
||||||
|
encryptedDataBuffer.getLength());
|
||||||
|
CryptoInputStream in = new CryptoInputStream(decryptedDataBuffer,
|
||||||
|
codec, bufferSize, key, iv);
|
||||||
|
DataInputStream dataIn = new DataInputStream(new BufferedInputStream(in));
|
||||||
|
|
||||||
|
// Check
|
||||||
|
DataInputBuffer originalData = new DataInputBuffer();
|
||||||
|
originalData.reset(data.getData(), 0, data.getLength());
|
||||||
|
DataInputStream originalIn = new DataInputStream(
|
||||||
|
new BufferedInputStream(originalData));
|
||||||
|
|
||||||
|
for(int i=0; i < count; ++i) {
|
||||||
|
RandomDatum k1 = new RandomDatum();
|
||||||
|
RandomDatum v1 = new RandomDatum();
|
||||||
|
k1.readFields(originalIn);
|
||||||
|
v1.readFields(originalIn);
|
||||||
|
|
||||||
|
RandomDatum k2 = new RandomDatum();
|
||||||
|
RandomDatum v2 = new RandomDatum();
|
||||||
|
k2.readFields(dataIn);
|
||||||
|
v2.readFields(dataIn);
|
||||||
|
assertTrue("original and encrypted-then-decrypted-output not equal",
|
||||||
|
k1.equals(k2) && v1.equals(v2));
|
||||||
|
|
||||||
|
// original and encrypted-then-decrypted-output have the same hashCode
|
||||||
|
Map<RandomDatum, String> m = new HashMap<RandomDatum, String>();
|
||||||
|
m.put(k1, k1.toString());
|
||||||
|
m.put(v1, v1.toString());
|
||||||
|
String result = m.get(k2);
|
||||||
|
assertEquals("k1 and k2 hashcode not equal", result, k1.toString());
|
||||||
|
result = m.get(v2);
|
||||||
|
assertEquals("v1 and v2 hashcode not equal", result, v1.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Decrypt data byte-at-a-time
|
||||||
|
originalData.reset(data.getData(), 0, data.getLength());
|
||||||
|
decryptedDataBuffer.reset(encryptedDataBuffer.getData(), 0,
|
||||||
|
encryptedDataBuffer.getLength());
|
||||||
|
in = new CryptoInputStream(decryptedDataBuffer,
|
||||||
|
codec, bufferSize, key, iv);
|
||||||
|
|
||||||
|
// Check
|
||||||
|
originalIn = new DataInputStream(new BufferedInputStream(originalData));
|
||||||
|
int expected;
|
||||||
|
do {
|
||||||
|
expected = originalIn.read();
|
||||||
|
assertEquals("Decrypted stream read by byte does not match",
|
||||||
|
expected, in.read());
|
||||||
|
} while (expected != -1);
|
||||||
|
|
||||||
|
LOG.info("SUCCESS! Completed checking " + count + " records");
|
||||||
|
|
||||||
|
// Check secure random generator
|
||||||
|
testSecureRandom(codec);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Test secure random generator */
|
||||||
|
private void testSecureRandom(CryptoCodec codec) {
|
||||||
|
// len = 16
|
||||||
|
checkSecureRandom(codec, 16);
|
||||||
|
// len = 32
|
||||||
|
checkSecureRandom(codec, 32);
|
||||||
|
// len = 128
|
||||||
|
checkSecureRandom(codec, 128);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void checkSecureRandom(CryptoCodec codec, int len) {
|
||||||
|
byte[] rand = new byte[len];
|
||||||
|
byte[] rand1 = new byte[len];
|
||||||
|
codec.generateSecureRandom(rand);
|
||||||
|
codec.generateSecureRandom(rand1);
|
||||||
|
|
||||||
|
Assert.assertEquals(len, rand.length);
|
||||||
|
Assert.assertEquals(len, rand1.length);
|
||||||
|
Assert.assertFalse(Arrays.equals(rand, rand1));
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,376 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.crypto;
|
||||||
|
|
||||||
|
import java.io.EOFException;
|
||||||
|
import java.io.FileDescriptor;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.io.OutputStream;
|
||||||
|
import java.nio.ByteBuffer;
|
||||||
|
import java.util.EnumSet;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.ByteBufferReadable;
|
||||||
|
import org.apache.hadoop.fs.CanSetDropBehind;
|
||||||
|
import org.apache.hadoop.fs.CanSetReadahead;
|
||||||
|
import org.apache.hadoop.fs.HasEnhancedByteBufferAccess;
|
||||||
|
import org.apache.hadoop.fs.HasFileDescriptor;
|
||||||
|
import org.apache.hadoop.fs.PositionedReadable;
|
||||||
|
import org.apache.hadoop.fs.ReadOption;
|
||||||
|
import org.apache.hadoop.fs.Seekable;
|
||||||
|
import org.apache.hadoop.fs.Syncable;
|
||||||
|
import org.apache.hadoop.io.ByteBufferPool;
|
||||||
|
import org.apache.hadoop.io.DataInputBuffer;
|
||||||
|
import org.apache.hadoop.io.DataOutputBuffer;
|
||||||
|
import org.junit.AfterClass;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
|
||||||
|
public class TestCryptoStreams extends CryptoStreamsTestBase {
|
||||||
|
/**
|
||||||
|
* Data storage.
|
||||||
|
* {@link #getOutputStream(int)} will write to this buf.
|
||||||
|
* {@link #getInputStream(int)} will read from this buf.
|
||||||
|
*/
|
||||||
|
private byte[] buf;
|
||||||
|
private int bufLen;
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void init() throws Exception {
|
||||||
|
Configuration conf = new Configuration();
|
||||||
|
codec = CryptoCodec.getInstance(conf);
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterClass
|
||||||
|
public static void shutdown() throws Exception {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected OutputStream getOutputStream(int bufferSize, byte[] key, byte[] iv)
|
||||||
|
throws IOException {
|
||||||
|
DataOutputBuffer out = new DataOutputBuffer() {
|
||||||
|
@Override
|
||||||
|
public void flush() throws IOException {
|
||||||
|
buf = getData();
|
||||||
|
bufLen = getLength();
|
||||||
|
}
|
||||||
|
@Override
|
||||||
|
public void close() throws IOException {
|
||||||
|
buf = getData();
|
||||||
|
bufLen = getLength();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
return new CryptoOutputStream(new FakeOutputStream(out),
|
||||||
|
codec, bufferSize, key, iv);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected InputStream getInputStream(int bufferSize, byte[] key, byte[] iv)
|
||||||
|
throws IOException {
|
||||||
|
DataInputBuffer in = new DataInputBuffer();
|
||||||
|
in.reset(buf, 0, bufLen);
|
||||||
|
return new CryptoInputStream(new FakeInputStream(in), codec, bufferSize,
|
||||||
|
key, iv);
|
||||||
|
}
|
||||||
|
|
||||||
|
private class FakeOutputStream extends OutputStream
|
||||||
|
implements Syncable, CanSetDropBehind{
|
||||||
|
private final byte[] oneByteBuf = new byte[1];
|
||||||
|
private final DataOutputBuffer out;
|
||||||
|
private boolean closed;
|
||||||
|
|
||||||
|
public FakeOutputStream(DataOutputBuffer out) {
|
||||||
|
this.out = out;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void write(byte b[], int off, int len) throws IOException {
|
||||||
|
if (b == null) {
|
||||||
|
throw new NullPointerException();
|
||||||
|
} else if (off < 0 || len < 0 || len > b.length - off) {
|
||||||
|
throw new IndexOutOfBoundsException();
|
||||||
|
} else if (len == 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
checkStream();
|
||||||
|
|
||||||
|
out.write(b, off, len);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void flush() throws IOException {
|
||||||
|
checkStream();
|
||||||
|
out.flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() throws IOException {
|
||||||
|
if (closed) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
out.close();
|
||||||
|
closed = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void write(int b) throws IOException {
|
||||||
|
oneByteBuf[0] = (byte)(b & 0xff);
|
||||||
|
write(oneByteBuf, 0, oneByteBuf.length);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setDropBehind(Boolean dropCache) throws IOException,
|
||||||
|
UnsupportedOperationException {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void hflush() throws IOException {
|
||||||
|
checkStream();
|
||||||
|
flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void hsync() throws IOException {
|
||||||
|
checkStream();
|
||||||
|
flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void checkStream() throws IOException {
|
||||||
|
if (closed) {
|
||||||
|
throw new IOException("Stream is closed!");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private class FakeInputStream extends InputStream implements
|
||||||
|
Seekable, PositionedReadable, ByteBufferReadable, HasFileDescriptor,
|
||||||
|
CanSetDropBehind, CanSetReadahead, HasEnhancedByteBufferAccess {
|
||||||
|
private final byte[] oneByteBuf = new byte[1];
|
||||||
|
private int pos = 0;
|
||||||
|
private final byte[] data;
|
||||||
|
private final int length;
|
||||||
|
private boolean closed = false;
|
||||||
|
|
||||||
|
public FakeInputStream(DataInputBuffer in) {
|
||||||
|
data = in.getData();
|
||||||
|
length = in.getLength();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void seek(long pos) throws IOException {
|
||||||
|
if (pos > length) {
|
||||||
|
throw new IOException("Cannot seek after EOF.");
|
||||||
|
}
|
||||||
|
if (pos < 0) {
|
||||||
|
throw new IOException("Cannot seek to negative offset.");
|
||||||
|
}
|
||||||
|
checkStream();
|
||||||
|
this.pos = (int)pos;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public long getPos() throws IOException {
|
||||||
|
return pos;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int available() throws IOException {
|
||||||
|
return length - pos;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int read(byte b[], int off, int len) throws IOException {
|
||||||
|
if (b == null) {
|
||||||
|
throw new NullPointerException();
|
||||||
|
} else if (off < 0 || len < 0 || len > b.length - off) {
|
||||||
|
throw new IndexOutOfBoundsException();
|
||||||
|
} else if (len == 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
checkStream();
|
||||||
|
|
||||||
|
if (pos < length) {
|
||||||
|
int n = (int) Math.min(len, length - pos);
|
||||||
|
System.arraycopy(data, pos, b, off, n);
|
||||||
|
pos += n;
|
||||||
|
return n;
|
||||||
|
}
|
||||||
|
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void checkStream() throws IOException {
|
||||||
|
if (closed) {
|
||||||
|
throw new IOException("Stream is closed!");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int read(ByteBuffer buf) throws IOException {
|
||||||
|
checkStream();
|
||||||
|
if (pos < length) {
|
||||||
|
int n = (int) Math.min(buf.remaining(), length - pos);
|
||||||
|
if (n > 0) {
|
||||||
|
buf.put(data, pos, n);
|
||||||
|
}
|
||||||
|
pos += n;
|
||||||
|
return n;
|
||||||
|
}
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public long skip(long n) throws IOException {
|
||||||
|
checkStream();
|
||||||
|
if ( n > 0 ) {
|
||||||
|
if( n + pos > length ) {
|
||||||
|
n = length - pos;
|
||||||
|
}
|
||||||
|
pos += n;
|
||||||
|
return n;
|
||||||
|
}
|
||||||
|
return n < 0 ? -1 : 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() throws IOException {
|
||||||
|
closed = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int read(long position, byte[] b, int off, int len)
|
||||||
|
throws IOException {
|
||||||
|
if (b == null) {
|
||||||
|
throw new NullPointerException();
|
||||||
|
} else if (off < 0 || len < 0 || len > b.length - off) {
|
||||||
|
throw new IndexOutOfBoundsException();
|
||||||
|
} else if (len == 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (position > length) {
|
||||||
|
throw new IOException("Cannot read after EOF.");
|
||||||
|
}
|
||||||
|
if (position < 0) {
|
||||||
|
throw new IOException("Cannot read to negative offset.");
|
||||||
|
}
|
||||||
|
|
||||||
|
checkStream();
|
||||||
|
|
||||||
|
if (position < length) {
|
||||||
|
int n = (int) Math.min(len, length - position);
|
||||||
|
System.arraycopy(data, (int)position, b, off, n);
|
||||||
|
return n;
|
||||||
|
}
|
||||||
|
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void readFully(long position, byte[] b, int off, int len)
|
||||||
|
throws IOException {
|
||||||
|
if (b == null) {
|
||||||
|
throw new NullPointerException();
|
||||||
|
} else if (off < 0 || len < 0 || len > b.length - off) {
|
||||||
|
throw new IndexOutOfBoundsException();
|
||||||
|
} else if (len == 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (position > length) {
|
||||||
|
throw new IOException("Cannot read after EOF.");
|
||||||
|
}
|
||||||
|
if (position < 0) {
|
||||||
|
throw new IOException("Cannot read to negative offset.");
|
||||||
|
}
|
||||||
|
|
||||||
|
checkStream();
|
||||||
|
|
||||||
|
if (position + len > length) {
|
||||||
|
throw new EOFException("Reach the end of stream.");
|
||||||
|
}
|
||||||
|
|
||||||
|
System.arraycopy(data, (int)position, b, off, len);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void readFully(long position, byte[] buffer) throws IOException {
|
||||||
|
readFully(position, buffer, 0, buffer.length);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ByteBuffer read(ByteBufferPool bufferPool, int maxLength,
|
||||||
|
EnumSet<ReadOption> opts) throws IOException,
|
||||||
|
UnsupportedOperationException {
|
||||||
|
if (bufferPool == null) {
|
||||||
|
throw new IOException("Please specify buffer pool.");
|
||||||
|
}
|
||||||
|
ByteBuffer buffer = bufferPool.getBuffer(true, maxLength);
|
||||||
|
int pos = buffer.position();
|
||||||
|
int n = read(buffer);
|
||||||
|
if (n >= 0) {
|
||||||
|
buffer.position(pos);
|
||||||
|
return buffer;
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void releaseBuffer(ByteBuffer buffer) {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setReadahead(Long readahead) throws IOException,
|
||||||
|
UnsupportedOperationException {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setDropBehind(Boolean dropCache) throws IOException,
|
||||||
|
UnsupportedOperationException {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public FileDescriptor getFileDescriptor() throws IOException {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean seekToNewSource(long targetPos) throws IOException {
|
||||||
|
if (targetPos > length) {
|
||||||
|
throw new IOException("Attempted to read past end of file.");
|
||||||
|
}
|
||||||
|
if (targetPos < 0) {
|
||||||
|
throw new IOException("Cannot seek after EOF.");
|
||||||
|
}
|
||||||
|
checkStream();
|
||||||
|
this.pos = (int)targetPos;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int read() throws IOException {
|
||||||
|
int ret = read( oneByteBuf, 0, 1 );
|
||||||
|
return ( ret <= 0 ) ? -1 : (oneByteBuf[0] & 0xff);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,120 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.crypto;
|
||||||
|
|
||||||
|
import static org.junit.Assert.assertTrue;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.io.OutputStream;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
|
||||||
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
|
import org.apache.hadoop.fs.FileUtil;
|
||||||
|
import org.apache.hadoop.fs.LocalFileSystem;
|
||||||
|
import org.apache.hadoop.fs.Path;
|
||||||
|
import org.junit.After;
|
||||||
|
import org.junit.AfterClass;
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
import org.junit.Ignore;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
public class TestCryptoStreamsForLocalFS extends CryptoStreamsTestBase {
|
||||||
|
private static final String TEST_ROOT_DIR
|
||||||
|
= System.getProperty("test.build.data","build/test/data") + "/work-dir/localfs";
|
||||||
|
|
||||||
|
private final File base = new File(TEST_ROOT_DIR);
|
||||||
|
private final Path file = new Path(TEST_ROOT_DIR, "test-file");
|
||||||
|
private static LocalFileSystem fileSys;
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void init() throws Exception {
|
||||||
|
Configuration conf = new Configuration();
|
||||||
|
conf = new Configuration(false);
|
||||||
|
conf.set("fs.file.impl", LocalFileSystem.class.getName());
|
||||||
|
fileSys = FileSystem.getLocal(conf);
|
||||||
|
conf.set(
|
||||||
|
CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX
|
||||||
|
+ CipherSuite.AES_CTR_NOPADDING.getConfigSuffix(),
|
||||||
|
OpensslAesCtrCryptoCodec.class.getName() + ","
|
||||||
|
+ JceAesCtrCryptoCodec.class.getName());
|
||||||
|
codec = CryptoCodec.getInstance(conf);
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterClass
|
||||||
|
public static void shutdown() throws Exception {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Before
|
||||||
|
@Override
|
||||||
|
public void setUp() throws IOException {
|
||||||
|
fileSys.delete(new Path(TEST_ROOT_DIR), true);
|
||||||
|
super.setUp();
|
||||||
|
}
|
||||||
|
|
||||||
|
@After
|
||||||
|
public void cleanUp() throws IOException {
|
||||||
|
FileUtil.setWritable(base, true);
|
||||||
|
FileUtil.fullyDelete(base);
|
||||||
|
assertTrue(!base.exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected OutputStream getOutputStream(int bufferSize, byte[] key, byte[] iv)
|
||||||
|
throws IOException {
|
||||||
|
return new CryptoOutputStream(fileSys.create(file), codec, bufferSize,
|
||||||
|
key, iv);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected InputStream getInputStream(int bufferSize, byte[] key, byte[] iv)
|
||||||
|
throws IOException {
|
||||||
|
return new CryptoInputStream(fileSys.open(file), codec, bufferSize,
|
||||||
|
key, iv);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Ignore("ChecksumFSInputChecker doesn't support ByteBuffer read")
|
||||||
|
@Override
|
||||||
|
@Test(timeout=1000)
|
||||||
|
public void testByteBufferRead() throws Exception {}
|
||||||
|
|
||||||
|
@Ignore("ChecksumFSOutputSummer doesn't support Syncable")
|
||||||
|
@Override
|
||||||
|
@Test(timeout=1000)
|
||||||
|
public void testSyncable() throws IOException {}
|
||||||
|
|
||||||
|
@Ignore("ChecksumFSInputChecker doesn't support ByteBuffer read")
|
||||||
|
@Override
|
||||||
|
@Test(timeout=1000)
|
||||||
|
public void testCombinedOp() throws Exception {}
|
||||||
|
|
||||||
|
@Ignore("ChecksumFSInputChecker doesn't support enhanced ByteBuffer access")
|
||||||
|
@Override
|
||||||
|
@Test(timeout=1000)
|
||||||
|
public void testHasEnhancedByteBufferAccess() throws Exception {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Ignore("ChecksumFSInputChecker doesn't support seekToNewSource")
|
||||||
|
@Override
|
||||||
|
@Test(timeout=1000)
|
||||||
|
public void testSeekToNewSource() throws Exception {
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,123 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.crypto;
|
||||||
|
|
||||||
|
import java.io.ByteArrayInputStream;
|
||||||
|
import java.io.ByteArrayOutputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.io.OutputStream;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.junit.AfterClass;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
import org.junit.Ignore;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test crypto streams using normal stream which does not support the
|
||||||
|
* additional interfaces that the Hadoop FileSystem streams implement
|
||||||
|
* (Seekable, PositionedReadable, ByteBufferReadable, HasFileDescriptor,
|
||||||
|
* CanSetDropBehind, CanSetReadahead, HasEnhancedByteBufferAccess, Syncable,
|
||||||
|
* CanSetDropBehind)
|
||||||
|
*/
|
||||||
|
public class TestCryptoStreamsNormal extends CryptoStreamsTestBase {
|
||||||
|
/**
|
||||||
|
* Data storage.
|
||||||
|
* {@link #getOutputStream(int, byte[], byte[])} will write to this buffer.
|
||||||
|
* {@link #getInputStream(int, byte[], byte[])} will read from this buffer.
|
||||||
|
*/
|
||||||
|
private byte[] buffer;
|
||||||
|
private int bufferLen;
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void init() throws Exception {
|
||||||
|
Configuration conf = new Configuration();
|
||||||
|
codec = CryptoCodec.getInstance(conf);
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterClass
|
||||||
|
public static void shutdown() throws Exception {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected OutputStream getOutputStream(int bufferSize, byte[] key, byte[] iv)
|
||||||
|
throws IOException {
|
||||||
|
OutputStream out = new ByteArrayOutputStream() {
|
||||||
|
@Override
|
||||||
|
public void flush() throws IOException {
|
||||||
|
buffer = buf;
|
||||||
|
bufferLen = count;
|
||||||
|
}
|
||||||
|
@Override
|
||||||
|
public void close() throws IOException {
|
||||||
|
buffer = buf;
|
||||||
|
bufferLen = count;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
return new CryptoOutputStream(out, codec, bufferSize, key, iv);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected InputStream getInputStream(int bufferSize, byte[] key, byte[] iv)
|
||||||
|
throws IOException {
|
||||||
|
ByteArrayInputStream in = new ByteArrayInputStream(buffer, 0, bufferLen);
|
||||||
|
return new CryptoInputStream(in, codec, bufferSize,
|
||||||
|
key, iv);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Ignore("Wrapped stream doesn't support Syncable")
|
||||||
|
@Override
|
||||||
|
@Test(timeout=1000)
|
||||||
|
public void testSyncable() throws IOException {}
|
||||||
|
|
||||||
|
@Ignore("Wrapped stream doesn't support PositionedRead")
|
||||||
|
@Override
|
||||||
|
@Test(timeout=1000)
|
||||||
|
public void testPositionedRead() throws IOException {}
|
||||||
|
|
||||||
|
@Ignore("Wrapped stream doesn't support ReadFully")
|
||||||
|
@Override
|
||||||
|
@Test(timeout=1000)
|
||||||
|
public void testReadFully() throws IOException {}
|
||||||
|
|
||||||
|
@Ignore("Wrapped stream doesn't support Seek")
|
||||||
|
@Override
|
||||||
|
@Test(timeout=1000)
|
||||||
|
public void testSeek() throws IOException {}
|
||||||
|
|
||||||
|
@Ignore("Wrapped stream doesn't support ByteBufferRead")
|
||||||
|
@Override
|
||||||
|
@Test(timeout=1000)
|
||||||
|
public void testByteBufferRead() throws IOException {}
|
||||||
|
|
||||||
|
@Ignore("Wrapped stream doesn't support ByteBufferRead, Seek")
|
||||||
|
@Override
|
||||||
|
@Test(timeout=1000)
|
||||||
|
public void testCombinedOp() throws IOException {}
|
||||||
|
|
||||||
|
@Ignore("Wrapped stream doesn't support SeekToNewSource")
|
||||||
|
@Override
|
||||||
|
@Test(timeout=1000)
|
||||||
|
public void testSeekToNewSource() throws IOException {}
|
||||||
|
|
||||||
|
@Ignore("Wrapped stream doesn't support HasEnhancedByteBufferAccess")
|
||||||
|
@Override
|
||||||
|
@Test(timeout=1000)
|
||||||
|
public void testHasEnhancedByteBufferAccess() throws IOException {}
|
||||||
|
}
|
|
@ -0,0 +1,31 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.crypto;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
|
||||||
|
public class TestCryptoStreamsWithOpensslAesCtrCryptoCodec
|
||||||
|
extends TestCryptoStreams {
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void init() throws Exception {
|
||||||
|
Configuration conf = new Configuration();
|
||||||
|
codec = CryptoCodec.getInstance(conf);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,110 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.crypto;
|
||||||
|
|
||||||
|
import java.nio.ByteBuffer;
|
||||||
|
import java.security.NoSuchAlgorithmException;
|
||||||
|
|
||||||
|
import javax.crypto.NoSuchPaddingException;
|
||||||
|
import javax.crypto.ShortBufferException;
|
||||||
|
|
||||||
|
import org.apache.hadoop.test.GenericTestUtils;
|
||||||
|
import org.junit.Assume;
|
||||||
|
import org.junit.Assert;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
public class TestOpensslCipher {
|
||||||
|
private static final byte[] key = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
|
||||||
|
0x07, 0x08, 0x09, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16};
|
||||||
|
private static final byte[] iv = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
|
||||||
|
0x07, 0x08, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08};
|
||||||
|
|
||||||
|
@Test(timeout=120000)
|
||||||
|
public void testGetInstance() throws Exception {
|
||||||
|
Assume.assumeTrue(OpensslCipher.getLoadingFailureReason() == null);
|
||||||
|
OpensslCipher cipher = OpensslCipher.getInstance("AES/CTR/NoPadding");
|
||||||
|
Assert.assertTrue(cipher != null);
|
||||||
|
|
||||||
|
try {
|
||||||
|
cipher = OpensslCipher.getInstance("AES2/CTR/NoPadding");
|
||||||
|
Assert.fail("Should specify correct algorithm.");
|
||||||
|
} catch (NoSuchAlgorithmException e) {
|
||||||
|
// Expect NoSuchAlgorithmException
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
cipher = OpensslCipher.getInstance("AES/CTR/NoPadding2");
|
||||||
|
Assert.fail("Should specify correct padding.");
|
||||||
|
} catch (NoSuchPaddingException e) {
|
||||||
|
// Expect NoSuchPaddingException
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout=120000)
|
||||||
|
public void testUpdateArguments() throws Exception {
|
||||||
|
Assume.assumeTrue(OpensslCipher.getLoadingFailureReason() == null);
|
||||||
|
OpensslCipher cipher = OpensslCipher.getInstance("AES/CTR/NoPadding");
|
||||||
|
Assert.assertTrue(cipher != null);
|
||||||
|
|
||||||
|
cipher.init(OpensslCipher.ENCRYPT_MODE, key, iv);
|
||||||
|
|
||||||
|
// Require direct buffers
|
||||||
|
ByteBuffer input = ByteBuffer.allocate(1024);
|
||||||
|
ByteBuffer output = ByteBuffer.allocate(1024);
|
||||||
|
|
||||||
|
try {
|
||||||
|
cipher.update(input, output);
|
||||||
|
Assert.fail("Input and output buffer should be direct buffer.");
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
GenericTestUtils.assertExceptionContains(
|
||||||
|
"Direct buffers are required", e);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Output buffer length should be sufficient to store output data
|
||||||
|
input = ByteBuffer.allocateDirect(1024);
|
||||||
|
output = ByteBuffer.allocateDirect(1000);
|
||||||
|
try {
|
||||||
|
cipher.update(input, output);
|
||||||
|
Assert.fail("Output buffer length should be sufficient " +
|
||||||
|
"to store output data");
|
||||||
|
} catch (ShortBufferException e) {
|
||||||
|
GenericTestUtils.assertExceptionContains(
|
||||||
|
"Output buffer is not sufficient", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout=120000)
|
||||||
|
public void testDoFinalArguments() throws Exception {
|
||||||
|
Assume.assumeTrue(OpensslCipher.getLoadingFailureReason() == null);
|
||||||
|
OpensslCipher cipher = OpensslCipher.getInstance("AES/CTR/NoPadding");
|
||||||
|
Assert.assertTrue(cipher != null);
|
||||||
|
|
||||||
|
cipher.init(OpensslCipher.ENCRYPT_MODE, key, iv);
|
||||||
|
|
||||||
|
// Require direct buffer
|
||||||
|
ByteBuffer output = ByteBuffer.allocate(1024);
|
||||||
|
|
||||||
|
try {
|
||||||
|
cipher.doFinal(output);
|
||||||
|
Assert.fail("Output buffer should be direct buffer.");
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
GenericTestUtils.assertExceptionContains(
|
||||||
|
"Direct buffer is required", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,114 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.crypto.random;
|
||||||
|
|
||||||
|
import java.util.Arrays;
|
||||||
|
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
public class TestOpensslSecureRandom {
|
||||||
|
|
||||||
|
@Test(timeout=120000)
|
||||||
|
public void testRandomBytes() throws Exception {
|
||||||
|
OpensslSecureRandom random = new OpensslSecureRandom();
|
||||||
|
|
||||||
|
// len = 16
|
||||||
|
checkRandomBytes(random, 16);
|
||||||
|
// len = 32
|
||||||
|
checkRandomBytes(random, 32);
|
||||||
|
// len = 128
|
||||||
|
checkRandomBytes(random, 128);
|
||||||
|
// len = 256
|
||||||
|
checkRandomBytes(random, 256);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test will timeout if secure random implementation always returns a
|
||||||
|
* constant value.
|
||||||
|
*/
|
||||||
|
private void checkRandomBytes(OpensslSecureRandom random, int len) {
|
||||||
|
byte[] bytes = new byte[len];
|
||||||
|
byte[] bytes1 = new byte[len];
|
||||||
|
random.nextBytes(bytes);
|
||||||
|
random.nextBytes(bytes1);
|
||||||
|
|
||||||
|
while (Arrays.equals(bytes, bytes1)) {
|
||||||
|
random.nextBytes(bytes1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test will timeout if secure random implementation always returns a
|
||||||
|
* constant value.
|
||||||
|
*/
|
||||||
|
@Test(timeout=120000)
|
||||||
|
public void testRandomInt() throws Exception {
|
||||||
|
OpensslSecureRandom random = new OpensslSecureRandom();
|
||||||
|
|
||||||
|
int rand1 = random.nextInt();
|
||||||
|
int rand2 = random.nextInt();
|
||||||
|
while (rand1 == rand2) {
|
||||||
|
rand2 = random.nextInt();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test will timeout if secure random implementation always returns a
|
||||||
|
* constant value.
|
||||||
|
*/
|
||||||
|
@Test(timeout=120000)
|
||||||
|
public void testRandomLong() throws Exception {
|
||||||
|
OpensslSecureRandom random = new OpensslSecureRandom();
|
||||||
|
|
||||||
|
long rand1 = random.nextLong();
|
||||||
|
long rand2 = random.nextLong();
|
||||||
|
while (rand1 == rand2) {
|
||||||
|
rand2 = random.nextLong();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test will timeout if secure random implementation always returns a
|
||||||
|
* constant value.
|
||||||
|
*/
|
||||||
|
@Test(timeout=120000)
|
||||||
|
public void testRandomFloat() throws Exception {
|
||||||
|
OpensslSecureRandom random = new OpensslSecureRandom();
|
||||||
|
|
||||||
|
float rand1 = random.nextFloat();
|
||||||
|
float rand2 = random.nextFloat();
|
||||||
|
while (rand1 == rand2) {
|
||||||
|
rand2 = random.nextFloat();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test will timeout if secure random implementation always returns a
|
||||||
|
* constant value.
|
||||||
|
*/
|
||||||
|
@Test(timeout=120000)
|
||||||
|
public void testRandomDouble() throws Exception {
|
||||||
|
OpensslSecureRandom random = new OpensslSecureRandom();
|
||||||
|
|
||||||
|
double rand1 = random.nextDouble();
|
||||||
|
double rand2 = random.nextDouble();
|
||||||
|
while (rand1 == rand2) {
|
||||||
|
rand2 = random.nextDouble();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,139 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.crypto.random;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Arrays;
|
||||||
|
|
||||||
|
import org.apache.commons.lang.SystemUtils;
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.junit.Assume;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
public class TestOsSecureRandom {
|
||||||
|
|
||||||
|
private static OsSecureRandom getOsSecureRandom() throws IOException {
|
||||||
|
Assume.assumeTrue(SystemUtils.IS_OS_LINUX);
|
||||||
|
OsSecureRandom random = new OsSecureRandom();
|
||||||
|
random.setConf(new Configuration());
|
||||||
|
return random;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout=120000)
|
||||||
|
public void testRandomBytes() throws Exception {
|
||||||
|
OsSecureRandom random = getOsSecureRandom();
|
||||||
|
// len = 16
|
||||||
|
checkRandomBytes(random, 16);
|
||||||
|
// len = 32
|
||||||
|
checkRandomBytes(random, 32);
|
||||||
|
// len = 128
|
||||||
|
checkRandomBytes(random, 128);
|
||||||
|
// len = 256
|
||||||
|
checkRandomBytes(random, 256);
|
||||||
|
random.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test will timeout if secure random implementation always returns a
|
||||||
|
* constant value.
|
||||||
|
*/
|
||||||
|
private void checkRandomBytes(OsSecureRandom random, int len) {
|
||||||
|
byte[] bytes = new byte[len];
|
||||||
|
byte[] bytes1 = new byte[len];
|
||||||
|
random.nextBytes(bytes);
|
||||||
|
random.nextBytes(bytes1);
|
||||||
|
|
||||||
|
while (Arrays.equals(bytes, bytes1)) {
|
||||||
|
random.nextBytes(bytes1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test will timeout if secure random implementation always returns a
|
||||||
|
* constant value.
|
||||||
|
*/
|
||||||
|
@Test(timeout=120000)
|
||||||
|
public void testRandomInt() throws Exception {
|
||||||
|
OsSecureRandom random = getOsSecureRandom();
|
||||||
|
|
||||||
|
int rand1 = random.nextInt();
|
||||||
|
int rand2 = random.nextInt();
|
||||||
|
while (rand1 == rand2) {
|
||||||
|
rand2 = random.nextInt();
|
||||||
|
}
|
||||||
|
random.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test will timeout if secure random implementation always returns a
|
||||||
|
* constant value.
|
||||||
|
*/
|
||||||
|
@Test(timeout=120000)
|
||||||
|
public void testRandomLong() throws Exception {
|
||||||
|
OsSecureRandom random = getOsSecureRandom();
|
||||||
|
|
||||||
|
long rand1 = random.nextLong();
|
||||||
|
long rand2 = random.nextLong();
|
||||||
|
while (rand1 == rand2) {
|
||||||
|
rand2 = random.nextLong();
|
||||||
|
}
|
||||||
|
random.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test will timeout if secure random implementation always returns a
|
||||||
|
* constant value.
|
||||||
|
*/
|
||||||
|
@Test(timeout=120000)
|
||||||
|
public void testRandomFloat() throws Exception {
|
||||||
|
OsSecureRandom random = getOsSecureRandom();
|
||||||
|
|
||||||
|
float rand1 = random.nextFloat();
|
||||||
|
float rand2 = random.nextFloat();
|
||||||
|
while (rand1 == rand2) {
|
||||||
|
rand2 = random.nextFloat();
|
||||||
|
}
|
||||||
|
random.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test will timeout if secure random implementation always returns a
|
||||||
|
* constant value.
|
||||||
|
*/
|
||||||
|
@Test(timeout=120000)
|
||||||
|
public void testRandomDouble() throws Exception {
|
||||||
|
OsSecureRandom random = getOsSecureRandom();
|
||||||
|
|
||||||
|
double rand1 = random.nextDouble();
|
||||||
|
double rand2 = random.nextDouble();
|
||||||
|
while (rand1 == rand2) {
|
||||||
|
rand2 = random.nextDouble();
|
||||||
|
}
|
||||||
|
random.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout=120000)
|
||||||
|
public void testRefillReservoir() throws Exception {
|
||||||
|
OsSecureRandom random = getOsSecureRandom();
|
||||||
|
|
||||||
|
for (int i = 0; i < 8196; i++) {
|
||||||
|
random.nextLong();
|
||||||
|
}
|
||||||
|
random.close();
|
||||||
|
}
|
||||||
|
}
|
|
@ -22,6 +22,7 @@ import static org.junit.Assert.*;
|
||||||
|
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
import org.apache.hadoop.crypto.OpensslCipher;
|
||||||
import org.apache.hadoop.io.compress.Lz4Codec;
|
import org.apache.hadoop.io.compress.Lz4Codec;
|
||||||
import org.apache.hadoop.io.compress.SnappyCodec;
|
import org.apache.hadoop.io.compress.SnappyCodec;
|
||||||
import org.apache.hadoop.io.compress.zlib.ZlibFactory;
|
import org.apache.hadoop.io.compress.zlib.ZlibFactory;
|
||||||
|
@ -54,6 +55,9 @@ public class TestNativeCodeLoader {
|
||||||
if (NativeCodeLoader.buildSupportsSnappy()) {
|
if (NativeCodeLoader.buildSupportsSnappy()) {
|
||||||
assertFalse(SnappyCodec.getLibraryName().isEmpty());
|
assertFalse(SnappyCodec.getLibraryName().isEmpty());
|
||||||
}
|
}
|
||||||
|
if (NativeCodeLoader.buildSupportsOpenssl()) {
|
||||||
|
assertFalse(OpensslCipher.getLibraryName().isEmpty());
|
||||||
|
}
|
||||||
assertFalse(Lz4Codec.getLibraryName().isEmpty());
|
assertFalse(Lz4Codec.getLibraryName().isEmpty());
|
||||||
LOG.info("TestNativeCodeLoader: libhadoop.so is loaded.");
|
LOG.info("TestNativeCodeLoader: libhadoop.so is loaded.");
|
||||||
}
|
}
|
||||||
|
|
|
@ -324,7 +324,23 @@
|
||||||
</comparator>
|
</comparator>
|
||||||
<comparator>
|
<comparator>
|
||||||
<type>RegexpComparator</type>
|
<type>RegexpComparator</type>
|
||||||
<expected-output>^\s*permission. Passing -f overwrites the destination if it already exists.( )*</expected-output>
|
<expected-output>^( |\t)*permission. Passing -f overwrites the destination if it already exists. raw( )*</expected-output>
|
||||||
|
</comparator>
|
||||||
|
<comparator>
|
||||||
|
<type>RegexpComparator</type>
|
||||||
|
<expected-output>^( |\t)*namespace extended attributes are preserved if \(1\) they are supported \(HDFS( )*</expected-output>
|
||||||
|
</comparator>
|
||||||
|
<comparator>
|
||||||
|
<type>RegexpComparator</type>
|
||||||
|
<expected-output>^( |\t)*only\) and, \(2\) all of the source and target pathnames are in the \/\.reserved\/raw( )*</expected-output>
|
||||||
|
</comparator>
|
||||||
|
<comparator>
|
||||||
|
<type>RegexpComparator</type>
|
||||||
|
<expected-output>^( |\t)*hierarchy. raw namespace xattr preservation is determined solely by the presence( )*</expected-output>
|
||||||
|
</comparator>
|
||||||
|
<comparator>
|
||||||
|
<type>RegexpComparator</type>
|
||||||
|
<expected-output>^\s*\(or absence\) of the \/\.reserved\/raw prefix and not by the -p option.( )*</expected-output>
|
||||||
</comparator>
|
</comparator>
|
||||||
</comparators>
|
</comparators>
|
||||||
</test>
|
</test>
|
||||||
|
|
|
@ -278,6 +278,97 @@ Trunk (Unreleased)
|
||||||
HDFS-6657. Remove link to 'Legacy UI' in trunk's Namenode UI.
|
HDFS-6657. Remove link to 'Legacy UI' in trunk's Namenode UI.
|
||||||
(Vinayakumar B via wheat 9)
|
(Vinayakumar B via wheat 9)
|
||||||
|
|
||||||
|
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
|
||||||
|
|
||||||
|
HDFS-6387. HDFS CLI admin tool for creating & deleting an
|
||||||
|
encryption zone. (clamb)
|
||||||
|
|
||||||
|
HDFS-6386. HDFS Encryption Zones (clamb)
|
||||||
|
|
||||||
|
HDFS-6388. HDFS integration with KeyProvider. (clamb)
|
||||||
|
|
||||||
|
HDFS-6473. Protocol and API for Encryption Zones (clamb)
|
||||||
|
|
||||||
|
HDFS-6392. Wire crypto streams for encrypted files in
|
||||||
|
DFSClient. (clamb and yliu)
|
||||||
|
|
||||||
|
HDFS-6476. Print out the KeyProvider after finding KP successfully on
|
||||||
|
startup. (Juan Yu via wang)
|
||||||
|
|
||||||
|
HDFS-6391. Get the Key/IV from the NameNode for encrypted files in
|
||||||
|
DFSClient. (Charles Lamb and wang)
|
||||||
|
|
||||||
|
HDFS-6389. Rename restrictions for encryption zones. (clamb)
|
||||||
|
|
||||||
|
HDFS-6605. Client server negotiation of cipher suite. (wang)
|
||||||
|
|
||||||
|
HDFS-6625. Remove the Delete Encryption Zone function (clamb)
|
||||||
|
|
||||||
|
HDFS-6516. List of Encryption Zones should be based on inodes (clamb)
|
||||||
|
|
||||||
|
HDFS-6629. Not able to create symlinks after HDFS-6516 (umamaheswararao)
|
||||||
|
|
||||||
|
HDFS-6635. Refactor encryption zone functionality into new
|
||||||
|
EncryptionZoneManager class. (wang)
|
||||||
|
|
||||||
|
HDFS-6474. Namenode needs to get the actual keys and iv from the
|
||||||
|
KeyProvider. (wang)
|
||||||
|
|
||||||
|
HDFS-6619. Clean up encryption-related tests. (wang)
|
||||||
|
|
||||||
|
HDFS-6405. Test Crypto streams in HDFS. (yliu via wang)
|
||||||
|
|
||||||
|
HDFS-6490. Fix the keyid format for generated keys in
|
||||||
|
FSNamesystem.createEncryptionZone (clamb)
|
||||||
|
|
||||||
|
HDFS-6716. Update usage of KeyProviderCryptoExtension APIs on NameNode.
|
||||||
|
(wang)
|
||||||
|
|
||||||
|
HDFS-6718. Remove EncryptionZoneManager lock. (wang)
|
||||||
|
|
||||||
|
HDFS-6720. Remove KeyProvider in EncryptionZoneManager. (wang)
|
||||||
|
|
||||||
|
HDFS-6738. Remove unnecessary getEncryptionZoneForPath call in
|
||||||
|
EZManager#createEncryptionZone. (clamb)
|
||||||
|
|
||||||
|
HDFS-6724. Decrypt EDEK before creating
|
||||||
|
CryptoInputStream/CryptoOutputStream. (wang)
|
||||||
|
|
||||||
|
HDFS-6509. Create a special /.reserved/raw directory for raw access to
|
||||||
|
encrypted data. (clamb via wang)
|
||||||
|
|
||||||
|
HDFS-6771. Require specification of an encryption key when creating
|
||||||
|
an encryption zone. (wang)
|
||||||
|
|
||||||
|
HDFS-6730. Create a .RAW extended attribute namespace. (clamb)
|
||||||
|
|
||||||
|
HDFS-6692. Add more HDFS encryption tests. (wang)
|
||||||
|
|
||||||
|
HDFS-6780. Batch the encryption zones listing API. (wang)
|
||||||
|
|
||||||
|
HDFS-6394. HDFS encryption documentation. (wang)
|
||||||
|
|
||||||
|
HDFS-6834. Improve the configuration guidance in DFSClient when there
|
||||||
|
are no Codec classes found in configs. (umamahesh)
|
||||||
|
|
||||||
|
HDFS-6546. Add non-superuser capability to get the encryption zone
|
||||||
|
for a specific path. (clamb)
|
||||||
|
|
||||||
|
HDFS-6733. Creating encryption zone results in NPE when
|
||||||
|
KeyProvider is null. (clamb)
|
||||||
|
|
||||||
|
HDFS-6785. Should not be able to create encryption zone using path
|
||||||
|
to a non-directory file. (clamb)
|
||||||
|
|
||||||
|
HDFS-6807. Fix TestReservedRawPaths. (clamb)
|
||||||
|
|
||||||
|
HDFS-6814. Mistakenly dfs.namenode.list.encryption.zones.num.responses configured
|
||||||
|
as boolean. (umamahesh)
|
||||||
|
|
||||||
|
HDFS-6817. Fix findbugs and other warnings. (yliu)
|
||||||
|
|
||||||
|
HDFS-6839. Fix TestCLI to expect new output. (clamb)
|
||||||
|
|
||||||
Release 2.6.0 - UNRELEASED
|
Release 2.6.0 - UNRELEASED
|
||||||
|
|
||||||
INCOMPATIBLE CHANGES
|
INCOMPATIBLE CHANGES
|
||||||
|
@ -428,6 +519,15 @@ Release 2.6.0 - UNRELEASED
|
||||||
HDFS-6188. An ip whitelist based implementation of TrustedChannelResolver.
|
HDFS-6188. An ip whitelist based implementation of TrustedChannelResolver.
|
||||||
(Benoy Antony via Arpit Agarwal)
|
(Benoy Antony via Arpit Agarwal)
|
||||||
|
|
||||||
|
HDFS-6858. Allow dfs.data.transfer.saslproperties.resolver.class default to
|
||||||
|
hadoop.security.saslproperties.resolver.class. (Benoy Antony via cnauroth)
|
||||||
|
|
||||||
|
HDFS-6878. Change MiniDFSCluster to support StorageType configuration
|
||||||
|
for individual directories. (Arpit Agarwal)
|
||||||
|
|
||||||
|
HDFS-6758. block writer should pass the expected block size to
|
||||||
|
DataXceiverServer. (Arpit Agarwal)
|
||||||
|
|
||||||
OPTIMIZATIONS
|
OPTIMIZATIONS
|
||||||
|
|
||||||
HDFS-6690. Deduplicate xattr names in memory. (wang)
|
HDFS-6690. Deduplicate xattr names in memory. (wang)
|
||||||
|
@ -543,7 +643,13 @@ Release 2.6.0 - UNRELEASED
|
||||||
HDFS-6569. OOB message can't be sent to the client when DataNode shuts down for upgrade
|
HDFS-6569. OOB message can't be sent to the client when DataNode shuts down for upgrade
|
||||||
(brandonli)
|
(brandonli)
|
||||||
|
|
||||||
Release 2.5.0 - UNRELEASED
|
HDFS-6868. portmap and nfs3 are documented as hadoop commands instead of hdfs
|
||||||
|
(brandonli)
|
||||||
|
|
||||||
|
HDFS-6870. Blocks and INodes could leak for Rename with overwrite flag. (Yi
|
||||||
|
Liu via jing9)
|
||||||
|
|
||||||
|
Release 2.5.0 - 2014-08-11
|
||||||
|
|
||||||
INCOMPATIBLE CHANGES
|
INCOMPATIBLE CHANGES
|
||||||
|
|
||||||
|
|
|
@ -304,6 +304,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
<include>datatransfer.proto</include>
|
<include>datatransfer.proto</include>
|
||||||
<include>fsimage.proto</include>
|
<include>fsimage.proto</include>
|
||||||
<include>hdfs.proto</include>
|
<include>hdfs.proto</include>
|
||||||
|
<include>encryption.proto</include>
|
||||||
</includes>
|
</includes>
|
||||||
</source>
|
</source>
|
||||||
<output>${project.build.directory}/generated-sources/java</output>
|
<output>${project.build.directory}/generated-sources/java</output>
|
||||||
|
|
|
@ -57,9 +57,9 @@ excludeFilenameRemote=$("$HADOOP_PREFIX/bin/hdfs" getconf -excludeFile)
|
||||||
|
|
||||||
if [ "$excludeFilenameRemote" = '' ] ; then
|
if [ "$excludeFilenameRemote" = '' ] ; then
|
||||||
echo \
|
echo \
|
||||||
"Error: hdfs getconf -excludeFile returned empty string, " \
|
"Error: hdfs getconf -excludeFile returned empty string, " \
|
||||||
"please setup dfs.hosts.exclude in hdfs-site.xml in local cluster " \
|
"please setup dfs.hosts.exclude in hdfs-site.xml in local cluster " \
|
||||||
"configuration and on all namenodes"
|
"configuration and on all namenodes"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
|
@ -15,250 +15,241 @@
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
# Environment Variables
|
function hadoop_usage
|
||||||
#
|
{
|
||||||
# JSVC_HOME home directory of jsvc binary. Required for starting secure
|
echo "Usage: hdfs [--config confdir] [--daemon (start|stop|status)] COMMAND"
|
||||||
# datanode.
|
|
||||||
#
|
|
||||||
# JSVC_OUTFILE path to jsvc output file. Defaults to
|
|
||||||
# $HADOOP_LOG_DIR/jsvc.out.
|
|
||||||
#
|
|
||||||
# JSVC_ERRFILE path to jsvc error file. Defaults to $HADOOP_LOG_DIR/jsvc.err.
|
|
||||||
|
|
||||||
bin=`which $0`
|
|
||||||
bin=`dirname ${bin}`
|
|
||||||
bin=`cd "$bin" > /dev/null; pwd`
|
|
||||||
|
|
||||||
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
|
|
||||||
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
|
|
||||||
. $HADOOP_LIBEXEC_DIR/hdfs-config.sh
|
|
||||||
|
|
||||||
function print_usage(){
|
|
||||||
echo "Usage: hdfs [--config confdir] COMMAND"
|
|
||||||
echo " where COMMAND is one of:"
|
echo " where COMMAND is one of:"
|
||||||
echo " dfs run a filesystem command on the file systems supported in Hadoop."
|
|
||||||
echo " namenode -format format the DFS filesystem"
|
|
||||||
echo " secondarynamenode run the DFS secondary namenode"
|
|
||||||
echo " namenode run the DFS namenode"
|
|
||||||
echo " journalnode run the DFS journalnode"
|
|
||||||
echo " zkfc run the ZK Failover Controller daemon"
|
|
||||||
echo " datanode run a DFS datanode"
|
|
||||||
echo " dfsadmin run a DFS admin client"
|
|
||||||
echo " haadmin run a DFS HA admin client"
|
|
||||||
echo " fsck run a DFS filesystem checking utility"
|
|
||||||
echo " balancer run a cluster balancing utility"
|
echo " balancer run a cluster balancing utility"
|
||||||
echo " jmxget get JMX exported values from NameNode or DataNode."
|
echo " cacheadmin configure the HDFS cache"
|
||||||
echo " oiv apply the offline fsimage viewer to an fsimage"
|
echo " classpath prints the class path needed to get the"
|
||||||
echo " oiv_legacy apply the offline fsimage viewer to an legacy fsimage"
|
echo " Hadoop jar and the required libraries"
|
||||||
echo " oev apply the offline edits viewer to an edits file"
|
echo " datanode run a DFS datanode"
|
||||||
|
echo " dfs run a filesystem command on the file system"
|
||||||
|
echo " dfsadmin run a DFS admin client"
|
||||||
echo " fetchdt fetch a delegation token from the NameNode"
|
echo " fetchdt fetch a delegation token from the NameNode"
|
||||||
|
echo " fsck run a DFS filesystem checking utility"
|
||||||
echo " getconf get config values from configuration"
|
echo " getconf get config values from configuration"
|
||||||
echo " groups get the groups which users belong to"
|
echo " groups get the groups which users belong to"
|
||||||
|
echo " haadmin run a DFS HA admin client"
|
||||||
|
echo " jmxget get JMX exported values from NameNode or DataNode."
|
||||||
|
echo " journalnode run the DFS journalnode"
|
||||||
|
echo " lsSnapshottableDir list all snapshottable dirs owned by the current user"
|
||||||
|
echo " Use -help to see options"
|
||||||
|
echo " namenode run the DFS namenode"
|
||||||
|
echo " Use -format to initialize the DFS filesystem"
|
||||||
|
echo " nfs3 run an NFS version 3 gateway"
|
||||||
|
echo " oev apply the offline edits viewer to an edits file"
|
||||||
|
echo " oiv apply the offline fsimage viewer to an fsimage"
|
||||||
|
echo " oiv_legacy apply the offline fsimage viewer to a legacy fsimage"
|
||||||
|
echo " portmap run a portmap service"
|
||||||
|
echo " secondarynamenode run the DFS secondary namenode"
|
||||||
echo " snapshotDiff diff two snapshots of a directory or diff the"
|
echo " snapshotDiff diff two snapshots of a directory or diff the"
|
||||||
echo " current directory contents with a snapshot"
|
echo " current directory contents with a snapshot"
|
||||||
echo " lsSnapshottableDir list all snapshottable dirs owned by the current user"
|
echo " zkfc run the ZK Failover Controller daemon"
|
||||||
echo " Use -help to see options"
|
echo " crypto configure HDFS encryption zones"
|
||||||
echo " portmap run a portmap service"
|
|
||||||
echo " nfs3 run an NFS version 3 gateway"
|
|
||||||
echo " cacheadmin configure the HDFS cache"
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "Most commands print help when invoked w/o parameters."
|
echo "Most commands print help when invoked w/o parameters."
|
||||||
}
|
}
|
||||||
|
|
||||||
if [ $# = 0 ]; then
|
# let's locate libexec...
|
||||||
print_usage
|
if [[ -n "${HADOOP_PREFIX}" ]]; then
|
||||||
exit
|
DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
|
||||||
|
else
|
||||||
|
this="${BASH_SOURCE-$0}"
|
||||||
|
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
|
||||||
|
DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
|
||||||
|
fi
|
||||||
|
|
||||||
|
HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
HADOOP_NEW_CONFIG=true
|
||||||
|
if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then
|
||||||
|
. "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh"
|
||||||
|
else
|
||||||
|
echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ $# = 0 ]]; then
|
||||||
|
hadoop_exit_with_usage 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
COMMAND=$1
|
COMMAND=$1
|
||||||
shift
|
shift
|
||||||
|
|
||||||
case $COMMAND in
|
case ${COMMAND} in
|
||||||
# usage flags
|
balancer)
|
||||||
--help|-help|-h)
|
CLASS=org.apache.hadoop.hdfs.server.balancer.Balancer
|
||||||
print_usage
|
HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_BALANCER_OPTS}"
|
||||||
|
;;
|
||||||
|
cacheadmin)
|
||||||
|
CLASS=org.apache.hadoop.hdfs.tools.CacheAdmin
|
||||||
|
;;
|
||||||
|
classpath)
|
||||||
|
hadoop_finalize
|
||||||
|
echo "${CLASSPATH}"
|
||||||
exit
|
exit
|
||||||
;;
|
;;
|
||||||
|
crypto)
|
||||||
|
CLASS=org.apache.hadoop.hdfs.tools.CryptoAdmin
|
||||||
|
;;
|
||||||
|
datanode)
|
||||||
|
daemon="true"
|
||||||
|
# Determine if we're starting a secure datanode, and
|
||||||
|
# if so, redefine appropriate variables
|
||||||
|
if [[ -n "${HADOOP_SECURE_DN_USER}" ]]; then
|
||||||
|
secure_service="true"
|
||||||
|
secure_user="${HADOOP_SECURE_DN_USER}"
|
||||||
|
|
||||||
|
# backward compatiblity
|
||||||
|
HADOOP_SECURE_PID_DIR="${HADOOP_SECURE_PID_DIR:-$HADOOP_SECURE_DN_PID_DIR}"
|
||||||
|
HADOOP_SECURE_LOG_DIR="${HADOOP_SECURE_LOG_DIR:-$HADOOP_SECURE_DN_LOG_DIR}"
|
||||||
|
|
||||||
|
HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_DN_SECURE_EXTRA_OPTS} ${HADOOP_DATANODE_OPTS}"
|
||||||
|
CLASS="org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter"
|
||||||
|
else
|
||||||
|
HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_DATANODE_OPTS}"
|
||||||
|
CLASS='org.apache.hadoop.hdfs.server.datanode.DataNode'
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
dfs)
|
||||||
|
CLASS=org.apache.hadoop.fs.FsShell
|
||||||
|
HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
|
||||||
|
;;
|
||||||
|
dfsadmin)
|
||||||
|
CLASS=org.apache.hadoop.hdfs.tools.DFSAdmin
|
||||||
|
HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
|
||||||
|
;;
|
||||||
|
fetchdt)
|
||||||
|
CLASS=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher
|
||||||
|
;;
|
||||||
|
fsck)
|
||||||
|
CLASS=org.apache.hadoop.hdfs.tools.DFSck
|
||||||
|
HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
|
||||||
|
;;
|
||||||
|
getconf)
|
||||||
|
CLASS=org.apache.hadoop.hdfs.tools.GetConf
|
||||||
|
;;
|
||||||
|
groups)
|
||||||
|
CLASS=org.apache.hadoop.hdfs.tools.GetGroups
|
||||||
|
;;
|
||||||
|
haadmin)
|
||||||
|
CLASS=org.apache.hadoop.hdfs.tools.DFSHAAdmin
|
||||||
|
CLASSPATH="${CLASSPATH}:${TOOL_PATH}"
|
||||||
|
HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
|
||||||
|
;;
|
||||||
|
journalnode)
|
||||||
|
daemon="true"
|
||||||
|
CLASS='org.apache.hadoop.hdfs.qjournal.server.JournalNode'
|
||||||
|
HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_JOURNALNODE_OPTS}"
|
||||||
|
;;
|
||||||
|
jmxget)
|
||||||
|
CLASS=org.apache.hadoop.hdfs.tools.JMXGet
|
||||||
|
;;
|
||||||
|
lsSnapshottableDir)
|
||||||
|
CLASS=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshottableDir
|
||||||
|
;;
|
||||||
|
namenode)
|
||||||
|
daemon="true"
|
||||||
|
CLASS='org.apache.hadoop.hdfs.server.namenode.NameNode'
|
||||||
|
HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NAMENODE_OPTS}"
|
||||||
|
;;
|
||||||
|
nfs3)
|
||||||
|
daemon="true"
|
||||||
|
if [[ -n "${HADOOP_PRIVILEGED_NFS_USER}" ]]; then
|
||||||
|
secure_service="true"
|
||||||
|
secure_user="${HADOOP_PRIVILEGED_NFS_USER}"
|
||||||
|
|
||||||
|
# backward compatiblity
|
||||||
|
HADOOP_SECURE_PID_DIR="${HADOOP_SECURE_PID_DIR:-$HADOOP_SECURE_NFS3_PID_DIR}"
|
||||||
|
HADOOP_SECURE_LOG_DIR="${HADOOP_SECURE_LOG_DIR:-$HADOOP_SECURE_NFS3_LOG_DIR}"
|
||||||
|
|
||||||
|
HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NFS3_SECURE_EXTRA_OPTS} ${HADOOP_NFS3_OPTS}"
|
||||||
|
CLASS=org.apache.hadoop.hdfs.nfs.nfs3.PrivilegedNfsGatewayStarter
|
||||||
|
else
|
||||||
|
HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NFS3_OPTS}"
|
||||||
|
CLASS=org.apache.hadoop.hdfs.nfs.nfs3.Nfs3
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
oev)
|
||||||
|
CLASS=org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer
|
||||||
|
;;
|
||||||
|
oiv)
|
||||||
|
CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewerPB
|
||||||
|
;;
|
||||||
|
oiv_legacy)
|
||||||
|
CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer
|
||||||
|
;;
|
||||||
|
portmap)
|
||||||
|
daemon="true"
|
||||||
|
CLASS=org.apache.hadoop.portmap.Portmap
|
||||||
|
HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_PORTMAP_OPTS}"
|
||||||
|
;;
|
||||||
|
secondarynamenode)
|
||||||
|
daemon="true"
|
||||||
|
CLASS='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode'
|
||||||
|
HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_SECONDARYNAMENODE_OPTS}"
|
||||||
|
;;
|
||||||
|
snapshotDiff)
|
||||||
|
CLASS=org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff
|
||||||
|
;;
|
||||||
|
zkfc)
|
||||||
|
daemon="true"
|
||||||
|
CLASS='org.apache.hadoop.hdfs.tools.DFSZKFailoverController'
|
||||||
|
HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_ZKFC_OPTS}"
|
||||||
|
;;
|
||||||
|
-*)
|
||||||
|
hadoop_exit_with_usage 1
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
CLASS="${COMMAND}"
|
||||||
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
# Determine if we're starting a secure datanode, and if so, redefine appropriate variables
|
if [[ -n "${secure_service}" ]]; then
|
||||||
if [ "$COMMAND" == "datanode" ] && [ "$EUID" -eq 0 ] && [ -n "$HADOOP_SECURE_DN_USER" ]; then
|
HADOOP_SECURE_USER="${secure_user}"
|
||||||
if [ -n "$JSVC_HOME" ]; then
|
if hadoop_verify_secure_prereq; then
|
||||||
if [ -n "$HADOOP_SECURE_DN_PID_DIR" ]; then
|
hadoop_setup_secure_service
|
||||||
HADOOP_PID_DIR=$HADOOP_SECURE_DN_PID_DIR
|
priv_outfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND-$HOSTNAME}.out"
|
||||||
fi
|
priv_errfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND-$HOSTNAME}.err"
|
||||||
|
priv_pidfile="${HADOOP_PID_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND-$HOSTNAME}.pid"
|
||||||
if [ -n "$HADOOP_SECURE_DN_LOG_DIR" ]; then
|
daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out"
|
||||||
HADOOP_LOG_DIR=$HADOOP_SECURE_DN_LOG_DIR
|
daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${COMMAND}.pid"
|
||||||
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR"
|
|
||||||
fi
|
|
||||||
|
|
||||||
HADOOP_IDENT_STRING=$HADOOP_SECURE_DN_USER
|
|
||||||
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.id.str=$HADOOP_IDENT_STRING"
|
|
||||||
starting_secure_dn="true"
|
|
||||||
else
|
|
||||||
echo "It looks like you're trying to start a secure DN, but \$JSVC_HOME"\
|
|
||||||
"isn't set. Falling back to starting insecure DN."
|
|
||||||
fi
|
fi
|
||||||
fi
|
|
||||||
|
|
||||||
# Determine if we're starting a privileged NFS daemon, and if so, redefine appropriate variables
|
|
||||||
if [ "$COMMAND" == "nfs3" ] && [ "$EUID" -eq 0 ] && [ -n "$HADOOP_PRIVILEGED_NFS_USER" ]; then
|
|
||||||
if [ -n "$JSVC_HOME" ]; then
|
|
||||||
if [ -n "$HADOOP_PRIVILEGED_NFS_PID_DIR" ]; then
|
|
||||||
HADOOP_PID_DIR=$HADOOP_PRIVILEGED_NFS_PID_DIR
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -n "$HADOOP_PRIVILEGED_NFS_LOG_DIR" ]; then
|
|
||||||
HADOOP_LOG_DIR=$HADOOP_PRIVILEGED_NFS_LOG_DIR
|
|
||||||
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR"
|
|
||||||
fi
|
|
||||||
|
|
||||||
HADOOP_IDENT_STRING=$HADOOP_PRIVILEGED_NFS_USER
|
|
||||||
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.id.str=$HADOOP_IDENT_STRING"
|
|
||||||
starting_privileged_nfs="true"
|
|
||||||
else
|
|
||||||
echo "It looks like you're trying to start a privileged NFS server, but"\
|
|
||||||
"\$JSVC_HOME isn't set. Falling back to starting unprivileged NFS server."
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ "$COMMAND" = "namenode" ] ; then
|
|
||||||
CLASS='org.apache.hadoop.hdfs.server.namenode.NameNode'
|
|
||||||
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_NAMENODE_OPTS"
|
|
||||||
elif [ "$COMMAND" = "zkfc" ] ; then
|
|
||||||
CLASS='org.apache.hadoop.hdfs.tools.DFSZKFailoverController'
|
|
||||||
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_ZKFC_OPTS"
|
|
||||||
elif [ "$COMMAND" = "secondarynamenode" ] ; then
|
|
||||||
CLASS='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode'
|
|
||||||
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_SECONDARYNAMENODE_OPTS"
|
|
||||||
elif [ "$COMMAND" = "datanode" ] ; then
|
|
||||||
CLASS='org.apache.hadoop.hdfs.server.datanode.DataNode'
|
|
||||||
if [ "$starting_secure_dn" = "true" ]; then
|
|
||||||
HADOOP_OPTS="$HADOOP_OPTS -jvm server $HADOOP_DATANODE_OPTS"
|
|
||||||
else
|
|
||||||
HADOOP_OPTS="$HADOOP_OPTS -server $HADOOP_DATANODE_OPTS"
|
|
||||||
fi
|
|
||||||
elif [ "$COMMAND" = "journalnode" ] ; then
|
|
||||||
CLASS='org.apache.hadoop.hdfs.qjournal.server.JournalNode'
|
|
||||||
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_JOURNALNODE_OPTS"
|
|
||||||
elif [ "$COMMAND" = "dfs" ] ; then
|
|
||||||
CLASS=org.apache.hadoop.fs.FsShell
|
|
||||||
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
|
|
||||||
elif [ "$COMMAND" = "dfsadmin" ] ; then
|
|
||||||
CLASS=org.apache.hadoop.hdfs.tools.DFSAdmin
|
|
||||||
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
|
|
||||||
elif [ "$COMMAND" = "haadmin" ] ; then
|
|
||||||
CLASS=org.apache.hadoop.hdfs.tools.DFSHAAdmin
|
|
||||||
CLASSPATH=${CLASSPATH}:${TOOL_PATH}
|
|
||||||
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
|
|
||||||
elif [ "$COMMAND" = "fsck" ] ; then
|
|
||||||
CLASS=org.apache.hadoop.hdfs.tools.DFSck
|
|
||||||
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
|
|
||||||
elif [ "$COMMAND" = "balancer" ] ; then
|
|
||||||
CLASS=org.apache.hadoop.hdfs.server.balancer.Balancer
|
|
||||||
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_BALANCER_OPTS"
|
|
||||||
elif [ "$COMMAND" = "jmxget" ] ; then
|
|
||||||
CLASS=org.apache.hadoop.hdfs.tools.JMXGet
|
|
||||||
elif [ "$COMMAND" = "oiv" ] ; then
|
|
||||||
CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewerPB
|
|
||||||
elif [ "$COMMAND" = "oiv_legacy" ] ; then
|
|
||||||
CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer
|
|
||||||
elif [ "$COMMAND" = "oev" ] ; then
|
|
||||||
CLASS=org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer
|
|
||||||
elif [ "$COMMAND" = "fetchdt" ] ; then
|
|
||||||
CLASS=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher
|
|
||||||
elif [ "$COMMAND" = "getconf" ] ; then
|
|
||||||
CLASS=org.apache.hadoop.hdfs.tools.GetConf
|
|
||||||
elif [ "$COMMAND" = "groups" ] ; then
|
|
||||||
CLASS=org.apache.hadoop.hdfs.tools.GetGroups
|
|
||||||
elif [ "$COMMAND" = "snapshotDiff" ] ; then
|
|
||||||
CLASS=org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff
|
|
||||||
elif [ "$COMMAND" = "lsSnapshottableDir" ] ; then
|
|
||||||
CLASS=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshottableDir
|
|
||||||
elif [ "$COMMAND" = "portmap" ] ; then
|
|
||||||
CLASS=org.apache.hadoop.portmap.Portmap
|
|
||||||
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_PORTMAP_OPTS"
|
|
||||||
elif [ "$COMMAND" = "nfs3" ] ; then
|
|
||||||
CLASS=org.apache.hadoop.hdfs.nfs.nfs3.Nfs3
|
|
||||||
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_NFS3_OPTS"
|
|
||||||
elif [ "$COMMAND" = "cacheadmin" ] ; then
|
|
||||||
CLASS=org.apache.hadoop.hdfs.tools.CacheAdmin
|
|
||||||
else
|
else
|
||||||
CLASS="$COMMAND"
|
daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out"
|
||||||
|
daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}.pid"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
export CLASSPATH=$CLASSPATH
|
if [[ "${HADOOP_DAEMON_MODE}" != "default" ]]; then
|
||||||
|
# shellcheck disable=SC2034
|
||||||
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,NullAppender}"
|
HADOOP_ROOT_LOGGER="${HADOOP_DAEMON_ROOT_LOGGER}"
|
||||||
|
if [[ -n "${secure_service}" ]]; then
|
||||||
# Check to see if we should start a secure datanode
|
# shellcheck disable=SC2034
|
||||||
if [ "$starting_secure_dn" = "true" ]; then
|
HADOOP_LOGFILE="hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.log"
|
||||||
if [ "$HADOOP_PID_DIR" = "" ]; then
|
|
||||||
HADOOP_SECURE_DN_PID="/tmp/hadoop_secure_dn.pid"
|
|
||||||
else
|
else
|
||||||
HADOOP_SECURE_DN_PID="$HADOOP_PID_DIR/hadoop_secure_dn.pid"
|
# shellcheck disable=SC2034
|
||||||
|
HADOOP_LOGFILE="hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.log"
|
||||||
fi
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
JSVC=$JSVC_HOME/jsvc
|
hadoop_add_param HADOOP_OPTS Xmx "${JAVA_HEAP_MAX}"
|
||||||
if [ ! -f $JSVC ]; then
|
hadoop_finalize
|
||||||
echo "JSVC_HOME is not set correctly so jsvc cannot be found. jsvc is required to run secure datanodes. "
|
|
||||||
echo "Please download and install jsvc from http://archive.apache.org/dist/commons/daemon/binaries/ "\
|
|
||||||
"and set JSVC_HOME to the directory containing the jsvc binary."
|
|
||||||
exit
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ ! $JSVC_OUTFILE ]]; then
|
export CLASSPATH
|
||||||
JSVC_OUTFILE="$HADOOP_LOG_DIR/jsvc.out"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ ! $JSVC_ERRFILE ]]; then
|
if [[ -n "${daemon}" ]]; then
|
||||||
JSVC_ERRFILE="$HADOOP_LOG_DIR/jsvc.err"
|
if [[ -n "${secure_service}" ]]; then
|
||||||
fi
|
hadoop_secure_daemon_handler \
|
||||||
|
"${HADOOP_DAEMON_MODE}" "${COMMAND}" "${CLASS}"\
|
||||||
exec "$JSVC" \
|
"${daemon_pidfile}" "${daemon_outfile}" \
|
||||||
-Dproc_$COMMAND -outfile "$JSVC_OUTFILE" \
|
"${priv_pidfile}" "${priv_outfile}" "${priv_errfile}" "$@"
|
||||||
-errfile "$JSVC_ERRFILE" \
|
|
||||||
-pidfile "$HADOOP_SECURE_DN_PID" \
|
|
||||||
-nodetach \
|
|
||||||
-user "$HADOOP_SECURE_DN_USER" \
|
|
||||||
-cp "$CLASSPATH" \
|
|
||||||
$JAVA_HEAP_MAX $HADOOP_OPTS \
|
|
||||||
org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter "$@"
|
|
||||||
elif [ "$starting_privileged_nfs" = "true" ] ; then
|
|
||||||
if [ "$HADOOP_PID_DIR" = "" ]; then
|
|
||||||
HADOOP_PRIVILEGED_NFS_PID="/tmp/hadoop_privileged_nfs3.pid"
|
|
||||||
else
|
else
|
||||||
HADOOP_PRIVILEGED_NFS_PID="$HADOOP_PID_DIR/hadoop_privileged_nfs3.pid"
|
hadoop_daemon_handler "${HADOOP_DAEMON_MODE}" "${COMMAND}" "${CLASS}"\
|
||||||
|
"${daemon_pidfile}" "${daemon_outfile}" "$@"
|
||||||
fi
|
fi
|
||||||
|
exit $?
|
||||||
JSVC=$JSVC_HOME/jsvc
|
|
||||||
if [ ! -f $JSVC ]; then
|
|
||||||
echo "JSVC_HOME is not set correctly so jsvc cannot be found. jsvc is required to run privileged NFS gateways. "
|
|
||||||
echo "Please download and install jsvc from http://archive.apache.org/dist/commons/daemon/binaries/ "\
|
|
||||||
"and set JSVC_HOME to the directory containing the jsvc binary."
|
|
||||||
exit
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ ! $JSVC_OUTFILE ]]; then
|
|
||||||
JSVC_OUTFILE="$HADOOP_LOG_DIR/nfs3_jsvc.out"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ ! $JSVC_ERRFILE ]]; then
|
|
||||||
JSVC_ERRFILE="$HADOOP_LOG_DIR/nfs3_jsvc.err"
|
|
||||||
fi
|
|
||||||
|
|
||||||
exec "$JSVC" \
|
|
||||||
-Dproc_$COMMAND -outfile "$JSVC_OUTFILE" \
|
|
||||||
-errfile "$JSVC_ERRFILE" \
|
|
||||||
-pidfile "$HADOOP_PRIVILEGED_NFS_PID" \
|
|
||||||
-nodetach \
|
|
||||||
-user "$HADOOP_PRIVILEGED_NFS_USER" \
|
|
||||||
-cp "$CLASSPATH" \
|
|
||||||
$JAVA_HEAP_MAX $HADOOP_OPTS \
|
|
||||||
org.apache.hadoop.hdfs.nfs.nfs3.PrivilegedNfsGatewayStarter "$@"
|
|
||||||
else
|
else
|
||||||
# run it
|
# shellcheck disable=SC2086
|
||||||
exec "$JAVA" -Dproc_$COMMAND $JAVA_HEAP_MAX $HADOOP_OPTS $CLASS "$@"
|
hadoop_java_exec "${COMMAND}" "${CLASS}" "$@"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
|
@ -18,19 +18,67 @@
|
||||||
# included in all the hdfs scripts with source command
|
# included in all the hdfs scripts with source command
|
||||||
# should not be executed directly
|
# should not be executed directly
|
||||||
|
|
||||||
bin=`which "$0"`
|
function hadoop_subproject_init
|
||||||
bin=`dirname "${bin}"`
|
{
|
||||||
bin=`cd "$bin"; pwd`
|
if [ -e "${HADOOP_CONF_DIR}/hdfs-env.sh" ]; then
|
||||||
|
. "${HADOOP_CONF_DIR}/hdfs-env.sh"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# at some point in time, someone thought it would be a good idea to
|
||||||
|
# create separate vars for every subproject. *sigh*
|
||||||
|
# let's perform some overrides and setup some defaults for bw compat
|
||||||
|
# this way the common hadoop var's == subproject vars and can be
|
||||||
|
# used interchangeable from here on out
|
||||||
|
# ...
|
||||||
|
# this should get deprecated at some point.
|
||||||
|
HADOOP_LOG_DIR="${HADOOP_HDFS_LOG_DIR:-$HADOOP_LOG_DIR}"
|
||||||
|
HADOOP_HDFS_LOG_DIR="${HADOOP_LOG_DIR}"
|
||||||
|
|
||||||
|
HADOOP_LOGFILE="${HADOOP_HDFS_LOGFILE:-$HADOOP_LOGFILE}"
|
||||||
|
HADOOP_HDFS_LOGFILE="${HADOOP_LOGFILE}"
|
||||||
|
|
||||||
|
HADOOP_NICENESS=${HADOOP_HDFS_NICENESS:-$HADOOP_NICENESS}
|
||||||
|
HADOOP_HDFS_NICENESS="${HADOOP_NICENESS}"
|
||||||
|
|
||||||
|
HADOOP_STOP_TIMEOUT=${HADOOP_HDFS_STOP_TIMEOUT:-$HADOOP_STOP_TIMEOUT}
|
||||||
|
HADOOP_HDFS_STOP_TIMEOUT="${HADOOP_STOP_TIMEOUT}"
|
||||||
|
|
||||||
|
HADOOP_PID_DIR="${HADOOP_HDFS_PID_DIR:-$HADOOP_PID_DIR}"
|
||||||
|
HADOOP_HDFS_PID_DIR="${HADOOP_PID_DIR}"
|
||||||
|
|
||||||
|
HADOOP_ROOT_LOGGER=${HADOOP_HDFS_ROOT_LOGGER:-$HADOOP_ROOT_LOGGER}
|
||||||
|
HADOOP_HDFS_ROOT_LOGGER="${HADOOP_ROOT_LOGGER}"
|
||||||
|
|
||||||
|
HADOOP_HDFS_HOME="${HADOOP_HDFS_HOME:-$HADOOP_HOME_DIR}"
|
||||||
|
|
||||||
|
HADOOP_IDENT_STRING="${HADOOP_HDFS_IDENT_STRING:-$HADOOP_IDENT_STRING}"
|
||||||
|
HADOOP_HDFS_IDENT_STRING="${HADOOP_IDENT_STRING}"
|
||||||
|
|
||||||
|
# turn on the defaults
|
||||||
|
|
||||||
|
export HADOOP_NAMENODE_OPTS=${HADOOP_NAMENODE_OPTS:-"-Dhadoop.security.logger=INFO,RFAS -Dhdfs.audit.logger=INFO,NullAppender"}
|
||||||
|
export HADOOP_SECONDARYNAMENODE_OPTS=${HADOOP_SECONDARYNAMENODE_OPTS:-"-Dhadoop.security.logger=INFO,RFAS -Dhdfs.audit.logger=INFO,NullAppender"}
|
||||||
|
export HADOOP_DATANODE_OPTS=${HADOOP_DATANODE_OPTS:-"-Dhadoop.security.logger=ERROR,RFAS"}
|
||||||
|
export HADOOP_DN_SECURE_EXTRA_OPTS=${HADOOP_DN_SECURE_EXTRA_OPTS:-"-jvm server"}
|
||||||
|
export HADOOP_NFS3_SECURE_EXTRA_OPTS=${HADOOP_NFS3_SECURE_EXTRA_OPTS:-"-jvm server"}
|
||||||
|
export HADOOP_PORTMAP_OPTS=${HADOOP_PORTMAP_OPTS:-"-Xmx512m"}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
|
if [[ -z "${HADOOP_LIBEXEC_DIR}" ]]; then
|
||||||
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
|
_hd_this="${BASH_SOURCE-$0}"
|
||||||
if [ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]; then
|
HADOOP_LIBEXEC_DIR=$(cd -P -- "$(dirname -- "${_hd_this}")" >/dev/null && pwd -P)
|
||||||
. ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh
|
|
||||||
elif [ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]; then
|
|
||||||
. "$HADOOP_COMMON_HOME"/libexec/hadoop-config.sh
|
|
||||||
elif [ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]; then
|
|
||||||
. "$HADOOP_HOME"/libexec/hadoop-config.sh
|
|
||||||
else
|
|
||||||
echo "Hadoop common not found."
|
|
||||||
exit
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if [ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]; then
|
||||||
|
. "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
|
||||||
|
elif [ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]; then
|
||||||
|
. "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh"
|
||||||
|
elif [ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]; then
|
||||||
|
. "${HADOOP_HOME}/libexec/hadoop-config.sh"
|
||||||
|
else
|
||||||
|
echo "ERROR: Hadoop common not found." 2>&1
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
|
@ -20,24 +20,40 @@
|
||||||
# This script refreshes all namenodes, it's a simple wrapper
|
# This script refreshes all namenodes, it's a simple wrapper
|
||||||
# for dfsadmin to support multiple namenodes.
|
# for dfsadmin to support multiple namenodes.
|
||||||
|
|
||||||
bin=`dirname "$0"`
|
# let's locate libexec...
|
||||||
bin=`cd "$bin"; pwd`
|
if [[ -n "${HADOOP_PREFIX}" ]]; then
|
||||||
|
DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
|
||||||
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
|
|
||||||
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
|
|
||||||
. $HADOOP_LIBEXEC_DIR/hdfs-config.sh
|
|
||||||
|
|
||||||
namenodes=$("$HADOOP_PREFIX/bin/hdfs" getconf -nnRpcAddresses)
|
|
||||||
if [ "$?" != '0' ] ; then errorFlag='1' ;
|
|
||||||
else
|
else
|
||||||
for namenode in $namenodes ; do
|
this="${BASH_SOURCE-$0}"
|
||||||
echo "Refreshing namenode [$namenode]"
|
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
|
||||||
"$HADOOP_PREFIX/bin/hdfs" dfsadmin -fs hdfs://$namenode -refreshNodes
|
DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
|
||||||
if [ "$?" != '0' ] ; then errorFlag='1' ; fi
|
fi
|
||||||
|
|
||||||
|
HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
HADOOP_NEW_CONFIG=true
|
||||||
|
if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then
|
||||||
|
. "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh"
|
||||||
|
else
|
||||||
|
echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
namenodes=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -nnRpcAddresses)
|
||||||
|
if [[ "$?" != '0' ]] ; then
|
||||||
|
errorFlag='1' ;
|
||||||
|
else
|
||||||
|
for namenode in ${namenodes} ; do
|
||||||
|
echo "Refreshing namenode [${namenode}]"
|
||||||
|
"${HADOOP_HDFS_HOME}/bin/hdfs" dfsadmin \
|
||||||
|
-fs hdfs://${namenode} -refreshNodes
|
||||||
|
if [[ "$?" != '0' ]]; then
|
||||||
|
errorFlag='1'
|
||||||
|
fi
|
||||||
done
|
done
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ "$errorFlag" = '1' ] ; then
|
if [[ "${errorFlag}" = '1' ]] ; then
|
||||||
echo "Error: refresh of namenodes failed, see error messages above."
|
echo "Error: refresh of namenodes failed, see error messages above."
|
||||||
exit 1
|
exit 1
|
||||||
else
|
else
|
||||||
|
|
|
@ -15,13 +15,31 @@
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
bin=`dirname "${BASH_SOURCE-$0}"`
|
function usage
|
||||||
bin=`cd "$bin"; pwd`
|
{
|
||||||
|
echo "Usage: start-balancer.sh [--config confdir] [-policy <policy>] [-threshold <threshold>]"
|
||||||
|
}
|
||||||
|
|
||||||
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
|
this="${BASH_SOURCE-$0}"
|
||||||
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
|
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
|
||||||
. $HADOOP_LIBEXEC_DIR/hdfs-config.sh
|
|
||||||
|
# let's locate libexec...
|
||||||
|
if [[ -n "${HADOOP_PREFIX}" ]]; then
|
||||||
|
DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
|
||||||
|
else
|
||||||
|
DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
|
||||||
|
fi
|
||||||
|
|
||||||
|
HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
HADOOP_NEW_CONFIG=true
|
||||||
|
if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then
|
||||||
|
. "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh"
|
||||||
|
else
|
||||||
|
echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
# Start balancer daemon.
|
# Start balancer daemon.
|
||||||
|
|
||||||
"$HADOOP_PREFIX"/sbin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs start balancer $@
|
exec "${bin}/hadoop-daemon.sh" --config "${HADOOP_CONF_DIR}" start balancer "$@"
|
||||||
|
|
|
@ -20,98 +20,128 @@
|
||||||
# Optinally upgrade or rollback dfs state.
|
# Optinally upgrade or rollback dfs state.
|
||||||
# Run this on master node.
|
# Run this on master node.
|
||||||
|
|
||||||
usage="Usage: start-dfs.sh [-upgrade|-rollback] [other options such as -clusterId]"
|
function hadoop_usage
|
||||||
|
{
|
||||||
|
echo "Usage: start-dfs.sh [-upgrade|-rollback] [-clusterId]"
|
||||||
|
}
|
||||||
|
|
||||||
bin=`dirname "${BASH_SOURCE-$0}"`
|
this="${BASH_SOURCE-$0}"
|
||||||
bin=`cd "$bin"; pwd`
|
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
|
||||||
|
|
||||||
|
# let's locate libexec...
|
||||||
|
if [[ -n "${HADOOP_PREFIX}" ]]; then
|
||||||
|
DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
|
||||||
|
else
|
||||||
|
DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
|
||||||
|
fi
|
||||||
|
|
||||||
|
HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
HADOOP_NEW_CONFIG=true
|
||||||
|
if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then
|
||||||
|
. "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh"
|
||||||
|
else
|
||||||
|
echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
|
|
||||||
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
|
|
||||||
. $HADOOP_LIBEXEC_DIR/hdfs-config.sh
|
|
||||||
|
|
||||||
# get arguments
|
# get arguments
|
||||||
if [ $# -ge 1 ]; then
|
if [[ $# -ge 1 ]]; then
|
||||||
nameStartOpt="$1"
|
nameStartOpt="$1"
|
||||||
shift
|
shift
|
||||||
case "$nameStartOpt" in
|
case "$nameStartOpt" in
|
||||||
(-upgrade)
|
-upgrade)
|
||||||
;;
|
;;
|
||||||
(-rollback)
|
-rollback)
|
||||||
dataStartOpt="$nameStartOpt"
|
dataStartOpt="$nameStartOpt"
|
||||||
;;
|
;;
|
||||||
(*)
|
*)
|
||||||
echo $usage
|
hadoop_exit_with_usage 1
|
||||||
exit 1
|
;;
|
||||||
;;
|
esac
|
||||||
esac
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
||||||
#Add other possible options
|
#Add other possible options
|
||||||
nameStartOpt="$nameStartOpt $@"
|
nameStartOpt="$nameStartOpt $@"
|
||||||
|
|
||||||
#---------------------------------------------------------
|
#---------------------------------------------------------
|
||||||
# namenodes
|
# namenodes
|
||||||
|
|
||||||
NAMENODES=$($HADOOP_PREFIX/bin/hdfs getconf -namenodes)
|
NAMENODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -namenodes 2>/dev/null)
|
||||||
|
|
||||||
|
if [[ -z "${NAMENODES}" ]]; then
|
||||||
|
NAMENODES=$(hostname)
|
||||||
|
fi
|
||||||
|
|
||||||
echo "Starting namenodes on [$NAMENODES]"
|
echo "Starting namenodes on [$NAMENODES]"
|
||||||
|
|
||||||
"$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \
|
"${bin}/hadoop-daemons.sh" \
|
||||||
--config "$HADOOP_CONF_DIR" \
|
--config "${HADOOP_CONF_DIR}" \
|
||||||
--hostnames "$NAMENODES" \
|
--hostnames "${NAMENODES}" \
|
||||||
--script "$bin/hdfs" start namenode $nameStartOpt
|
start namenode ${nameStartOpt}
|
||||||
|
|
||||||
#---------------------------------------------------------
|
#---------------------------------------------------------
|
||||||
# datanodes (using default slaves file)
|
# datanodes (using default slaves file)
|
||||||
|
|
||||||
if [ -n "$HADOOP_SECURE_DN_USER" ]; then
|
if [[ -n "${HADOOP_SECURE_DN_USER}" ]] &&
|
||||||
echo \
|
[[ -z "${HADOOP_SECURE_COMMAND}" ]]; then
|
||||||
"Attempting to start secure cluster, skipping datanodes. " \
|
echo "ERROR: Attempting to start secure cluster, skipping datanodes. "
|
||||||
"Run start-secure-dns.sh as root to complete startup."
|
echo "Run start-secure-dns.sh as root or configure "
|
||||||
|
echo "\${HADOOP_SECURE_COMMAND} to complete startup."
|
||||||
else
|
else
|
||||||
"$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \
|
|
||||||
--config "$HADOOP_CONF_DIR" \
|
echo "Starting datanodes"
|
||||||
--script "$bin/hdfs" start datanode $dataStartOpt
|
|
||||||
|
"${bin}/hadoop-daemons.sh" \
|
||||||
|
--config "${HADOOP_CONF_DIR}" \
|
||||||
|
start datanode ${dataStartOpt}
|
||||||
fi
|
fi
|
||||||
|
|
||||||
#---------------------------------------------------------
|
#---------------------------------------------------------
|
||||||
# secondary namenodes (if any)
|
# secondary namenodes (if any)
|
||||||
|
|
||||||
SECONDARY_NAMENODES=$($HADOOP_PREFIX/bin/hdfs getconf -secondarynamenodes 2>/dev/null)
|
SECONDARY_NAMENODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -secondarynamenodes 2>/dev/null)
|
||||||
|
|
||||||
if [ -n "$SECONDARY_NAMENODES" ]; then
|
if [[ "${SECONDARY_NAMENODES}" == "0.0.0.0" ]]; then
|
||||||
echo "Starting secondary namenodes [$SECONDARY_NAMENODES]"
|
SECONDARY_NAMENODES=$(hostname)
|
||||||
|
fi
|
||||||
|
|
||||||
"$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \
|
if [[ -n "${SECONDARY_NAMENODES}" ]]; then
|
||||||
--config "$HADOOP_CONF_DIR" \
|
echo "Starting secondary namenodes [${SECONDARY_NAMENODES}]"
|
||||||
--hostnames "$SECONDARY_NAMENODES" \
|
|
||||||
--script "$bin/hdfs" start secondarynamenode
|
"${bin}/hadoop-daemons.sh" \
|
||||||
|
--config "${HADOOP_CONF_DIR}" \
|
||||||
|
--hostnames "${SECONDARY_NAMENODES}" \
|
||||||
|
start secondarynamenode
|
||||||
fi
|
fi
|
||||||
|
|
||||||
#---------------------------------------------------------
|
#---------------------------------------------------------
|
||||||
# quorumjournal nodes (if any)
|
# quorumjournal nodes (if any)
|
||||||
|
|
||||||
SHARED_EDITS_DIR=$($HADOOP_PREFIX/bin/hdfs getconf -confKey dfs.namenode.shared.edits.dir 2>&-)
|
SHARED_EDITS_DIR=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey dfs.namenode.shared.edits.dir 2>&-)
|
||||||
|
|
||||||
case "$SHARED_EDITS_DIR" in
|
case "${SHARED_EDITS_DIR}" in
|
||||||
qjournal://*)
|
qjournal://*)
|
||||||
JOURNAL_NODES=$(echo "$SHARED_EDITS_DIR" | sed 's,qjournal://\([^/]*\)/.*,\1,g; s/;/ /g; s/:[0-9]*//g')
|
JOURNAL_NODES=$(echo "${SHARED_EDITS_DIR}" | sed 's,qjournal://\([^/]*\)/.*,\1,g; s/;/ /g; s/:[0-9]*//g')
|
||||||
echo "Starting journal nodes [$JOURNAL_NODES]"
|
echo "Starting journal nodes [${JOURNAL_NODES}]"
|
||||||
"$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \
|
"${bin}/hadoop-daemons.sh" \
|
||||||
--config "$HADOOP_CONF_DIR" \
|
--config "${HADOOP_CONF_DIR}" \
|
||||||
--hostnames "$JOURNAL_NODES" \
|
--hostnames "${JOURNAL_NODES}" \
|
||||||
--script "$bin/hdfs" start journalnode ;;
|
start journalnode
|
||||||
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
#---------------------------------------------------------
|
#---------------------------------------------------------
|
||||||
# ZK Failover controllers, if auto-HA is enabled
|
# ZK Failover controllers, if auto-HA is enabled
|
||||||
AUTOHA_ENABLED=$($HADOOP_PREFIX/bin/hdfs getconf -confKey dfs.ha.automatic-failover.enabled)
|
AUTOHA_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey dfs.ha.automatic-failover.enabled | tr '[:upper:]' '[:lower:]')
|
||||||
if [ "$(echo "$AUTOHA_ENABLED" | tr A-Z a-z)" = "true" ]; then
|
if [[ "${AUTOHA_ENABLED}" = "true" ]]; then
|
||||||
echo "Starting ZK Failover Controllers on NN hosts [$NAMENODES]"
|
echo "Starting ZK Failover Controllers on NN hosts [${NAMENODES}]"
|
||||||
"$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \
|
"${bin}/hadoop-daemons.sh" \
|
||||||
--config "$HADOOP_CONF_DIR" \
|
--config "${HADOOP_CONF_DIR}" \
|
||||||
--hostnames "$NAMENODES" \
|
--hostnames "${NAMENODES}" \
|
||||||
--script "$bin/hdfs" start zkfc
|
start zkfc
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# eof
|
# eof
|
||||||
|
|
|
@ -17,17 +17,33 @@
|
||||||
|
|
||||||
# Run as root to start secure datanodes in a security-enabled cluster.
|
# Run as root to start secure datanodes in a security-enabled cluster.
|
||||||
|
|
||||||
usage="Usage (run as root in order to start secure datanodes): start-secure-dns.sh"
|
|
||||||
|
|
||||||
bin=`dirname "${BASH_SOURCE-$0}"`
|
function hadoop_usage {
|
||||||
bin=`cd "$bin"; pwd`
|
echo "Usage: start-secure-dns.sh"
|
||||||
|
}
|
||||||
|
|
||||||
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
|
this="${BASH_SOURCE-$0}"
|
||||||
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
|
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
|
||||||
. $HADOOP_LIBEXEC_DIR/hdfs-config.sh
|
|
||||||
|
|
||||||
if [ "$EUID" -eq 0 ] && [ -n "$HADOOP_SECURE_DN_USER" ]; then
|
# let's locate libexec...
|
||||||
"$HADOOP_PREFIX"/sbin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs start datanode $dataStartOpt
|
if [[ -n "${HADOOP_PREFIX}" ]]; then
|
||||||
|
DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
|
||||||
else
|
else
|
||||||
echo $usage
|
DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
|
||||||
|
fi
|
||||||
|
|
||||||
|
HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
HADOOP_NEW_CONFIG=true
|
||||||
|
if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then
|
||||||
|
. "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh"
|
||||||
|
else
|
||||||
|
echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "${EUID}" -eq 0 ]] && [[ -n "${HADOOP_SECURE_DN_USER}" ]]; then
|
||||||
|
exec "${bin}/hadoop-daemons.sh" --config "${HADOOP_CONF_DIR}" start datanode "${dataStartOpt}"
|
||||||
|
else
|
||||||
|
echo hadoop_usage_and_exit 1
|
||||||
fi
|
fi
|
||||||
|
|
|
@ -15,14 +15,32 @@
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
bin=`dirname "${BASH_SOURCE-$0}"`
|
function hadoop_usage
|
||||||
bin=`cd "$bin"; pwd`
|
{
|
||||||
|
echo "Usage: stop-balancer.sh [--config confdir]"
|
||||||
|
}
|
||||||
|
|
||||||
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
|
this="${BASH_SOURCE-$0}"
|
||||||
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
|
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
|
||||||
. $HADOOP_LIBEXEC_DIR/hdfs-config.sh
|
|
||||||
|
# let's locate libexec...
|
||||||
|
if [[ -n "${HADOOP_PREFIX}" ]]; then
|
||||||
|
DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
|
||||||
|
else
|
||||||
|
DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
|
||||||
|
fi
|
||||||
|
|
||||||
|
HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
HADOOP_NEW_CONFIG=true
|
||||||
|
if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then
|
||||||
|
. "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh"
|
||||||
|
else
|
||||||
|
echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
# Stop balancer daemon.
|
# Stop balancer daemon.
|
||||||
# Run this on the machine where the balancer is running
|
# Run this on the machine where the balancer is running
|
||||||
|
|
||||||
"$HADOOP_PREFIX"/sbin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs stop balancer
|
"${bin}/hadoop-daemon.sh" --config "${HADOOP_CONF_DIR}" stop balancer
|
||||||
|
|
|
@ -15,75 +15,100 @@
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
bin=`dirname "${BASH_SOURCE-$0}"`
|
function hadoop_usage
|
||||||
bin=`cd "$bin"; pwd`
|
{
|
||||||
|
echo "Usage: start-balancer.sh [--config confdir] [-policy <policy>] [-threshold <threshold>]"
|
||||||
|
}
|
||||||
|
|
||||||
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
|
this="${BASH_SOURCE-$0}"
|
||||||
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
|
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
|
||||||
. $HADOOP_LIBEXEC_DIR/hdfs-config.sh
|
|
||||||
|
# let's locate libexec...
|
||||||
|
if [[ -n "${HADOOP_PREFIX}" ]]; then
|
||||||
|
DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
|
||||||
|
else
|
||||||
|
DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
|
||||||
|
fi
|
||||||
|
|
||||||
|
HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
HADOOP_NEW_CONFIG=true
|
||||||
|
if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then
|
||||||
|
. "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh"
|
||||||
|
else
|
||||||
|
echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
#---------------------------------------------------------
|
#---------------------------------------------------------
|
||||||
# namenodes
|
# namenodes
|
||||||
|
|
||||||
NAMENODES=$($HADOOP_PREFIX/bin/hdfs getconf -namenodes)
|
NAMENODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -namenodes)
|
||||||
|
|
||||||
echo "Stopping namenodes on [$NAMENODES]"
|
echo "Stopping namenodes on [$NAMENODES]"
|
||||||
|
|
||||||
"$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \
|
"${bin}/hadoop-daemons.sh" \
|
||||||
--config "$HADOOP_CONF_DIR" \
|
--config "${HADOOP_CONF_DIR}" \
|
||||||
--hostnames "$NAMENODES" \
|
--hostnames "${NAMENODES}" \
|
||||||
--script "$bin/hdfs" stop namenode
|
stop namenode
|
||||||
|
|
||||||
#---------------------------------------------------------
|
#---------------------------------------------------------
|
||||||
# datanodes (using default slaves file)
|
# datanodes (using default slaves file)
|
||||||
|
|
||||||
if [ -n "$HADOOP_SECURE_DN_USER" ]; then
|
if [[ -n "${HADOOP_SECURE_DN_USER}" ]] &&
|
||||||
|
[[ -z "${HADOOP_SECURE_COMMAND}" ]]; then
|
||||||
echo \
|
echo \
|
||||||
"Attempting to stop secure cluster, skipping datanodes. " \
|
"ERROR: Attempting to stop secure cluster, skipping datanodes. " \
|
||||||
"Run stop-secure-dns.sh as root to complete shutdown."
|
"Run stop-secure-dns.sh as root to complete shutdown."
|
||||||
else
|
else
|
||||||
"$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \
|
|
||||||
--config "$HADOOP_CONF_DIR" \
|
echo "Stopping datanodes"
|
||||||
--script "$bin/hdfs" stop datanode
|
|
||||||
|
"${bin}/hadoop-daemons.sh" --config "${HADOOP_CONF_DIR}" stop datanode
|
||||||
fi
|
fi
|
||||||
|
|
||||||
#---------------------------------------------------------
|
#---------------------------------------------------------
|
||||||
# secondary namenodes (if any)
|
# secondary namenodes (if any)
|
||||||
|
|
||||||
SECONDARY_NAMENODES=$($HADOOP_PREFIX/bin/hdfs getconf -secondarynamenodes 2>/dev/null)
|
SECONDARY_NAMENODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -secondarynamenodes 2>/dev/null)
|
||||||
|
|
||||||
if [ -n "$SECONDARY_NAMENODES" ]; then
|
if [[ "${SECONDARY_NAMENODES}" == "0.0.0.0" ]]; then
|
||||||
echo "Stopping secondary namenodes [$SECONDARY_NAMENODES]"
|
SECONDARY_NAMENODES=$(hostname)
|
||||||
|
fi
|
||||||
|
|
||||||
"$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \
|
if [[ -n "${SECONDARY_NAMENODES}" ]]; then
|
||||||
--config "$HADOOP_CONF_DIR" \
|
echo "Stopping secondary namenodes [${SECONDARY_NAMENODES}]"
|
||||||
--hostnames "$SECONDARY_NAMENODES" \
|
|
||||||
--script "$bin/hdfs" stop secondarynamenode
|
"${bin}/hadoop-daemons.sh" \
|
||||||
|
--config "${HADOOP_CONF_DIR}" \
|
||||||
|
--hostnames "${SECONDARY_NAMENODES}" \
|
||||||
|
stop secondarynamenode
|
||||||
fi
|
fi
|
||||||
|
|
||||||
#---------------------------------------------------------
|
#---------------------------------------------------------
|
||||||
# quorumjournal nodes (if any)
|
# quorumjournal nodes (if any)
|
||||||
|
|
||||||
SHARED_EDITS_DIR=$($HADOOP_PREFIX/bin/hdfs getconf -confKey dfs.namenode.shared.edits.dir 2>&-)
|
SHARED_EDITS_DIR=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey dfs.namenode.shared.edits.dir 2>&-)
|
||||||
|
|
||||||
case "$SHARED_EDITS_DIR" in
|
case "${SHARED_EDITS_DIR}" in
|
||||||
qjournal://*)
|
qjournal://*)
|
||||||
JOURNAL_NODES=$(echo "$SHARED_EDITS_DIR" | sed 's,qjournal://\([^/]*\)/.*,\1,g; s/;/ /g; s/:[0-9]*//g')
|
JOURNAL_NODES=$(echo "${SHARED_EDITS_DIR}" | sed 's,qjournal://\([^/]*\)/.*,\1,g; s/;/ /g; s/:[0-9]*//g')
|
||||||
echo "Stopping journal nodes [$JOURNAL_NODES]"
|
echo "Stopping journal nodes [${JOURNAL_NODES}]"
|
||||||
"$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \
|
"${bin}/hadoop-daemons.sh" \
|
||||||
--config "$HADOOP_CONF_DIR" \
|
--config "${HADOOP_CONF_DIR}" \
|
||||||
--hostnames "$JOURNAL_NODES" \
|
--hostnames "${JOURNAL_NODES}" \
|
||||||
--script "$bin/hdfs" stop journalnode ;;
|
stop journalnode
|
||||||
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
#---------------------------------------------------------
|
#---------------------------------------------------------
|
||||||
# ZK Failover controllers, if auto-HA is enabled
|
# ZK Failover controllers, if auto-HA is enabled
|
||||||
AUTOHA_ENABLED=$($HADOOP_PREFIX/bin/hdfs getconf -confKey dfs.ha.automatic-failover.enabled)
|
AUTOHA_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey dfs.ha.automatic-failover.enabled | tr '[:upper:]' '[:lower:]')
|
||||||
if [ "$(echo "$AUTOHA_ENABLED" | tr A-Z a-z)" = "true" ]; then
|
if [[ "${AUTOHA_ENABLED}" = "true" ]]; then
|
||||||
echo "Stopping ZK Failover Controllers on NN hosts [$NAMENODES]"
|
echo "Stopping ZK Failover Controllers on NN hosts [${NAMENODES}]"
|
||||||
"$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \
|
"${bin}/hadoop-daemons.sh" \
|
||||||
--config "$HADOOP_CONF_DIR" \
|
--config "${HADOOP_CONF_DIR}" \
|
||||||
--hostnames "$NAMENODES" \
|
--hostnames "${NAMENODES}" \
|
||||||
--script "$bin/hdfs" stop zkfc
|
stop zkfc
|
||||||
fi
|
fi
|
||||||
# eof
|
# eof
|
||||||
|
|
|
@ -17,17 +17,33 @@
|
||||||
|
|
||||||
# Run as root to start secure datanodes in a security-enabled cluster.
|
# Run as root to start secure datanodes in a security-enabled cluster.
|
||||||
|
|
||||||
usage="Usage (run as root in order to stop secure datanodes): stop-secure-dns.sh"
|
|
||||||
|
|
||||||
bin=`dirname "${BASH_SOURCE-$0}"`
|
function hadoop_usage {
|
||||||
bin=`cd "$bin"; pwd`
|
echo "Usage (run as root in order to stop secure datanodes): stop-secure-dns.sh"
|
||||||
|
}
|
||||||
|
|
||||||
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
|
this="${BASH_SOURCE-$0}"
|
||||||
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
|
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
|
||||||
. $HADOOP_LIBEXEC_DIR/hdfs-config.sh
|
|
||||||
|
|
||||||
if [ "$EUID" -eq 0 ] && [ -n "$HADOOP_SECURE_DN_USER" ]; then
|
# let's locate libexec...
|
||||||
"$HADOOP_PREFIX"/sbin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs stop datanode
|
if [[ -n "${HADOOP_PREFIX}" ]]; then
|
||||||
|
DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
|
||||||
else
|
else
|
||||||
echo $usage
|
DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
|
||||||
|
fi
|
||||||
|
|
||||||
|
HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
HADOOP_NEW_CONFIG=true
|
||||||
|
if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then
|
||||||
|
. "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh"
|
||||||
|
else
|
||||||
|
echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "${EUID}" -eq 0 ]] && [[ -n "${HADOOP_SECURE_DN_USER}" ]]; then
|
||||||
|
"${bin}/hadoop-daemons.sh" --config "${HADOOP_CONF_DIR}" stop datanode
|
||||||
|
else
|
||||||
|
hadoop_exit_with_usage 1
|
||||||
fi
|
fi
|
||||||
|
|
|
@ -17,7 +17,6 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.hadoop.fs;
|
package org.apache.hadoop.fs;
|
||||||
|
|
||||||
|
|
||||||
import java.io.FileNotFoundException;
|
import java.io.FileNotFoundException;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
|
@ -31,6 +30,7 @@ import java.util.NoSuchElementException;
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.crypto.CryptoCodec;
|
||||||
import org.apache.hadoop.fs.permission.AclEntry;
|
import org.apache.hadoop.fs.permission.AclEntry;
|
||||||
import org.apache.hadoop.fs.permission.AclStatus;
|
import org.apache.hadoop.fs.permission.AclStatus;
|
||||||
import org.apache.hadoop.fs.permission.FsAction;
|
import org.apache.hadoop.fs.permission.FsAction;
|
||||||
|
@ -38,6 +38,8 @@ import org.apache.hadoop.fs.permission.FsPermission;
|
||||||
import org.apache.hadoop.fs.Options.ChecksumOpt;
|
import org.apache.hadoop.fs.Options.ChecksumOpt;
|
||||||
import org.apache.hadoop.hdfs.CorruptFileBlockIterator;
|
import org.apache.hadoop.hdfs.CorruptFileBlockIterator;
|
||||||
import org.apache.hadoop.hdfs.DFSClient;
|
import org.apache.hadoop.hdfs.DFSClient;
|
||||||
|
import org.apache.hadoop.hdfs.DFSInputStream;
|
||||||
|
import org.apache.hadoop.hdfs.DFSOutputStream;
|
||||||
import org.apache.hadoop.hdfs.HdfsConfiguration;
|
import org.apache.hadoop.hdfs.HdfsConfiguration;
|
||||||
import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
|
import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
|
||||||
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
|
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
|
||||||
|
@ -59,6 +61,7 @@ import org.apache.hadoop.util.Progressable;
|
||||||
public class Hdfs extends AbstractFileSystem {
|
public class Hdfs extends AbstractFileSystem {
|
||||||
|
|
||||||
DFSClient dfs;
|
DFSClient dfs;
|
||||||
|
final CryptoCodec factory;
|
||||||
private boolean verifyChecksum = true;
|
private boolean verifyChecksum = true;
|
||||||
|
|
||||||
static {
|
static {
|
||||||
|
@ -85,6 +88,7 @@ public class Hdfs extends AbstractFileSystem {
|
||||||
}
|
}
|
||||||
|
|
||||||
this.dfs = new DFSClient(theUri, conf, getStatistics());
|
this.dfs = new DFSClient(theUri, conf, getStatistics());
|
||||||
|
this.factory = CryptoCodec.getInstance(conf);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -97,9 +101,12 @@ public class Hdfs extends AbstractFileSystem {
|
||||||
EnumSet<CreateFlag> createFlag, FsPermission absolutePermission,
|
EnumSet<CreateFlag> createFlag, FsPermission absolutePermission,
|
||||||
int bufferSize, short replication, long blockSize, Progressable progress,
|
int bufferSize, short replication, long blockSize, Progressable progress,
|
||||||
ChecksumOpt checksumOpt, boolean createParent) throws IOException {
|
ChecksumOpt checksumOpt, boolean createParent) throws IOException {
|
||||||
return new HdfsDataOutputStream(dfs.primitiveCreate(getUriPath(f),
|
|
||||||
absolutePermission, createFlag, createParent, replication, blockSize,
|
final DFSOutputStream dfsos = dfs.primitiveCreate(getUriPath(f),
|
||||||
progress, bufferSize, checksumOpt), getStatistics());
|
absolutePermission, createFlag, createParent, replication, blockSize,
|
||||||
|
progress, bufferSize, checksumOpt);
|
||||||
|
return dfs.createWrappedOutputStream(dfsos, statistics,
|
||||||
|
dfsos.getInitialLen());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -308,8 +315,9 @@ public class Hdfs extends AbstractFileSystem {
|
||||||
@Override
|
@Override
|
||||||
public HdfsDataInputStream open(Path f, int bufferSize)
|
public HdfsDataInputStream open(Path f, int bufferSize)
|
||||||
throws IOException, UnresolvedLinkException {
|
throws IOException, UnresolvedLinkException {
|
||||||
return new DFSClient.DFSDataInputStream(dfs.open(getUriPath(f),
|
final DFSInputStream dfsis = dfs.open(getUriPath(f),
|
||||||
bufferSize, verifyChecksum));
|
bufferSize, verifyChecksum);
|
||||||
|
return dfs.createWrappedInputStream(dfsis);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -26,8 +26,8 @@ import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
/**
|
/**
|
||||||
* XAttr is the POSIX Extended Attribute model similar to that found in
|
* XAttr is the POSIX Extended Attribute model similar to that found in
|
||||||
* traditional Operating Systems. Extended Attributes consist of one
|
* traditional Operating Systems. Extended Attributes consist of one
|
||||||
* or more name/value pairs associated with a file or directory. Four
|
* or more name/value pairs associated with a file or directory. Five
|
||||||
* namespaces are defined: user, trusted, security and system.
|
* namespaces are defined: user, trusted, security, system and raw.
|
||||||
* 1) USER namespace attributes may be used by any user to store
|
* 1) USER namespace attributes may be used by any user to store
|
||||||
* arbitrary information. Access permissions in this namespace are
|
* arbitrary information. Access permissions in this namespace are
|
||||||
* defined by a file directory's permission bits. For sticky directories,
|
* defined by a file directory's permission bits. For sticky directories,
|
||||||
|
@ -43,6 +43,12 @@ import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
* <br>
|
* <br>
|
||||||
* 4) SECURITY namespace attributes are used by the fs kernel for
|
* 4) SECURITY namespace attributes are used by the fs kernel for
|
||||||
* security features. It is not visible to users.
|
* security features. It is not visible to users.
|
||||||
|
* <br>
|
||||||
|
* 5) RAW namespace attributes are used for internal system attributes that
|
||||||
|
* sometimes need to be exposed. Like SYSTEM namespace attributes they are
|
||||||
|
* not visible to the user except when getXAttr/getXAttrs is called on a file
|
||||||
|
* or directory in the /.reserved/raw HDFS directory hierarchy. These
|
||||||
|
* attributes can only be accessed by the superuser.
|
||||||
* <p/>
|
* <p/>
|
||||||
* @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
|
* @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
|
||||||
* http://en.wikipedia.org/wiki/Extended_file_attributes</a>
|
* http://en.wikipedia.org/wiki/Extended_file_attributes</a>
|
||||||
|
@ -55,7 +61,8 @@ public class XAttr {
|
||||||
USER,
|
USER,
|
||||||
TRUSTED,
|
TRUSTED,
|
||||||
SECURITY,
|
SECURITY,
|
||||||
SYSTEM;
|
SYSTEM,
|
||||||
|
RAW;
|
||||||
}
|
}
|
||||||
|
|
||||||
private final NameSpace ns;
|
private final NameSpace ns;
|
||||||
|
|
|
@ -17,6 +17,11 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.hadoop.hdfs;
|
package org.apache.hadoop.hdfs;
|
||||||
|
|
||||||
|
import static org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
|
||||||
|
import static org.apache.hadoop.crypto.key.KeyProviderCryptoExtension
|
||||||
|
.EncryptedKeyVersion;
|
||||||
|
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX;
|
||||||
|
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY;
|
||||||
import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_DEFAULT;
|
import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_DEFAULT;
|
||||||
import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY;
|
import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY;
|
||||||
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT;
|
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT;
|
||||||
|
@ -76,6 +81,7 @@ import java.net.Socket;
|
||||||
import java.net.SocketAddress;
|
import java.net.SocketAddress;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.net.UnknownHostException;
|
import java.net.UnknownHostException;
|
||||||
|
import java.security.GeneralSecurityException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.EnumSet;
|
import java.util.EnumSet;
|
||||||
|
@ -95,6 +101,11 @@ import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.crypto.CipherSuite;
|
||||||
|
import org.apache.hadoop.crypto.CryptoCodec;
|
||||||
|
import org.apache.hadoop.crypto.CryptoInputStream;
|
||||||
|
import org.apache.hadoop.crypto.CryptoOutputStream;
|
||||||
|
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
|
||||||
import org.apache.hadoop.fs.BlockLocation;
|
import org.apache.hadoop.fs.BlockLocation;
|
||||||
import org.apache.hadoop.fs.BlockStorageLocation;
|
import org.apache.hadoop.fs.BlockStorageLocation;
|
||||||
import org.apache.hadoop.fs.CacheFlag;
|
import org.apache.hadoop.fs.CacheFlag;
|
||||||
|
@ -102,6 +113,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
|
||||||
import org.apache.hadoop.fs.ContentSummary;
|
import org.apache.hadoop.fs.ContentSummary;
|
||||||
import org.apache.hadoop.fs.CreateFlag;
|
import org.apache.hadoop.fs.CreateFlag;
|
||||||
import org.apache.hadoop.fs.FileAlreadyExistsException;
|
import org.apache.hadoop.fs.FileAlreadyExistsException;
|
||||||
|
import org.apache.hadoop.fs.FileEncryptionInfo;
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
import org.apache.hadoop.fs.FsServerDefaults;
|
import org.apache.hadoop.fs.FsServerDefaults;
|
||||||
import org.apache.hadoop.fs.FsStatus;
|
import org.apache.hadoop.fs.FsStatus;
|
||||||
|
@ -140,6 +152,9 @@ import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
|
||||||
import org.apache.hadoop.hdfs.protocol.DatanodeID;
|
import org.apache.hadoop.hdfs.protocol.DatanodeID;
|
||||||
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
|
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
|
||||||
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
|
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.EncryptionZoneIterator;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.EncryptionZoneWithId;
|
||||||
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
|
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
|
||||||
import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
|
import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
|
||||||
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
|
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
|
||||||
|
@ -249,7 +264,11 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
|
||||||
private static final DFSHedgedReadMetrics HEDGED_READ_METRIC =
|
private static final DFSHedgedReadMetrics HEDGED_READ_METRIC =
|
||||||
new DFSHedgedReadMetrics();
|
new DFSHedgedReadMetrics();
|
||||||
private static ThreadPoolExecutor HEDGED_READ_THREAD_POOL;
|
private static ThreadPoolExecutor HEDGED_READ_THREAD_POOL;
|
||||||
|
private final CryptoCodec codec;
|
||||||
|
@VisibleForTesting
|
||||||
|
List<CipherSuite> cipherSuites;
|
||||||
|
@VisibleForTesting
|
||||||
|
KeyProviderCryptoExtension provider;
|
||||||
/**
|
/**
|
||||||
* DFSClient configuration
|
* DFSClient configuration
|
||||||
*/
|
*/
|
||||||
|
@ -581,7 +600,17 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
|
||||||
this.authority = nameNodeUri == null? "null": nameNodeUri.getAuthority();
|
this.authority = nameNodeUri == null? "null": nameNodeUri.getAuthority();
|
||||||
this.clientName = "DFSClient_" + dfsClientConf.taskId + "_" +
|
this.clientName = "DFSClient_" + dfsClientConf.taskId + "_" +
|
||||||
DFSUtil.getRandom().nextInt() + "_" + Thread.currentThread().getId();
|
DFSUtil.getRandom().nextInt() + "_" + Thread.currentThread().getId();
|
||||||
|
this.codec = CryptoCodec.getInstance(conf);
|
||||||
|
this.cipherSuites = Lists.newArrayListWithCapacity(1);
|
||||||
|
if (codec != null) {
|
||||||
|
cipherSuites.add(codec.getCipherSuite());
|
||||||
|
}
|
||||||
|
provider = DFSUtil.createKeyProviderCryptoExtension(conf);
|
||||||
|
if (provider == null) {
|
||||||
|
LOG.info("No KeyProvider found.");
|
||||||
|
} else {
|
||||||
|
LOG.info("Found KeyProvider: " + provider.toString());
|
||||||
|
}
|
||||||
int numResponseToDrop = conf.getInt(
|
int numResponseToDrop = conf.getInt(
|
||||||
DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY,
|
DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY,
|
||||||
DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_DEFAULT);
|
DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_DEFAULT);
|
||||||
|
@ -1280,7 +1309,93 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
|
||||||
|
|
||||||
return volumeBlockLocations;
|
return volumeBlockLocations;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decrypts a EDEK by consulting the KeyProvider.
|
||||||
|
*/
|
||||||
|
private KeyVersion decryptEncryptedDataEncryptionKey(FileEncryptionInfo
|
||||||
|
feInfo) throws IOException {
|
||||||
|
if (provider == null) {
|
||||||
|
throw new IOException("No KeyProvider is configured, cannot access" +
|
||||||
|
" an encrypted file");
|
||||||
|
}
|
||||||
|
EncryptedKeyVersion ekv = EncryptedKeyVersion.createForDecryption(
|
||||||
|
feInfo.getEzKeyVersionName(), feInfo.getIV(),
|
||||||
|
feInfo.getEncryptedDataEncryptionKey());
|
||||||
|
try {
|
||||||
|
return provider.decryptEncryptedKey(ekv);
|
||||||
|
} catch (GeneralSecurityException e) {
|
||||||
|
throw new IOException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wraps the stream in a CryptoInputStream if the underlying file is
|
||||||
|
* encrypted.
|
||||||
|
*/
|
||||||
|
public HdfsDataInputStream createWrappedInputStream(DFSInputStream dfsis)
|
||||||
|
throws IOException {
|
||||||
|
final FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo();
|
||||||
|
if (feInfo != null) {
|
||||||
|
// File is encrypted, wrap the stream in a crypto stream.
|
||||||
|
KeyVersion decrypted = decryptEncryptedDataEncryptionKey(feInfo);
|
||||||
|
CryptoCodec codec = CryptoCodec
|
||||||
|
.getInstance(conf, feInfo.getCipherSuite());
|
||||||
|
if (codec == null) {
|
||||||
|
throw new IOException("No configuration found for the cipher suite "
|
||||||
|
+ feInfo.getCipherSuite().getConfigSuffix() + " prefixed with "
|
||||||
|
+ HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX
|
||||||
|
+ ". Please see the example configuration "
|
||||||
|
+ "hadoop.security.crypto.codec.classes.EXAMPLECIPHERSUITE "
|
||||||
|
+ "at core-default.xml for details.");
|
||||||
|
}
|
||||||
|
final CryptoInputStream cryptoIn =
|
||||||
|
new CryptoInputStream(dfsis, codec, decrypted.getMaterial(),
|
||||||
|
feInfo.getIV());
|
||||||
|
return new HdfsDataInputStream(cryptoIn);
|
||||||
|
} else {
|
||||||
|
// No FileEncryptionInfo so no encryption.
|
||||||
|
return new HdfsDataInputStream(dfsis);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wraps the stream in a CryptoOutputStream if the underlying file is
|
||||||
|
* encrypted.
|
||||||
|
*/
|
||||||
|
public HdfsDataOutputStream createWrappedOutputStream(DFSOutputStream dfsos,
|
||||||
|
FileSystem.Statistics statistics) throws IOException {
|
||||||
|
return createWrappedOutputStream(dfsos, statistics, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wraps the stream in a CryptoOutputStream if the underlying file is
|
||||||
|
* encrypted.
|
||||||
|
*/
|
||||||
|
public HdfsDataOutputStream createWrappedOutputStream(DFSOutputStream dfsos,
|
||||||
|
FileSystem.Statistics statistics, long startPos) throws IOException {
|
||||||
|
final FileEncryptionInfo feInfo = dfsos.getFileEncryptionInfo();
|
||||||
|
if (feInfo != null) {
|
||||||
|
if (codec == null) {
|
||||||
|
throw new IOException("No configuration found for the cipher suite "
|
||||||
|
+ HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY + " value prefixed with "
|
||||||
|
+ HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX
|
||||||
|
+ ". Please see the example configuration "
|
||||||
|
+ "hadoop.security.crypto.codec.classes.EXAMPLECIPHERSUITE "
|
||||||
|
+ "at core-default.xml for details.");
|
||||||
|
}
|
||||||
|
// File is encrypted, wrap the stream in a crypto stream.
|
||||||
|
KeyVersion decrypted = decryptEncryptedDataEncryptionKey(feInfo);
|
||||||
|
final CryptoOutputStream cryptoOut =
|
||||||
|
new CryptoOutputStream(dfsos, codec,
|
||||||
|
decrypted.getMaterial(), feInfo.getIV(), startPos);
|
||||||
|
return new HdfsDataOutputStream(cryptoOut, statistics, startPos);
|
||||||
|
} else {
|
||||||
|
// No FileEncryptionInfo present so no encryption.
|
||||||
|
return new HdfsDataOutputStream(dfsos, statistics, startPos);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public DFSInputStream open(String src)
|
public DFSInputStream open(String src)
|
||||||
throws IOException, UnresolvedLinkException {
|
throws IOException, UnresolvedLinkException {
|
||||||
return open(src, dfsClientConf.ioBufferSize, true, null);
|
return open(src, dfsClientConf.ioBufferSize, true, null);
|
||||||
|
@ -1483,7 +1598,8 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
|
||||||
}
|
}
|
||||||
final DFSOutputStream result = DFSOutputStream.newStreamForCreate(this,
|
final DFSOutputStream result = DFSOutputStream.newStreamForCreate(this,
|
||||||
src, masked, flag, createParent, replication, blockSize, progress,
|
src, masked, flag, createParent, replication, blockSize, progress,
|
||||||
buffersize, dfsClientConf.createChecksum(checksumOpt), favoredNodeStrs);
|
buffersize, dfsClientConf.createChecksum(checksumOpt),
|
||||||
|
favoredNodeStrs, cipherSuites);
|
||||||
beginFileLease(result.getFileId(), result);
|
beginFileLease(result.getFileId(), result);
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
@ -1530,7 +1646,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
|
||||||
DataChecksum checksum = dfsClientConf.createChecksum(checksumOpt);
|
DataChecksum checksum = dfsClientConf.createChecksum(checksumOpt);
|
||||||
result = DFSOutputStream.newStreamForCreate(this, src, absPermission,
|
result = DFSOutputStream.newStreamForCreate(this, src, absPermission,
|
||||||
flag, createParent, replication, blockSize, progress, buffersize,
|
flag, createParent, replication, blockSize, progress, buffersize,
|
||||||
checksum);
|
checksum, null, cipherSuites);
|
||||||
}
|
}
|
||||||
beginFileLease(result.getFileId(), result);
|
beginFileLease(result.getFileId(), result);
|
||||||
return result;
|
return result;
|
||||||
|
@ -1608,7 +1724,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
|
||||||
final Progressable progress, final FileSystem.Statistics statistics
|
final Progressable progress, final FileSystem.Statistics statistics
|
||||||
) throws IOException {
|
) throws IOException {
|
||||||
final DFSOutputStream out = append(src, buffersize, progress);
|
final DFSOutputStream out = append(src, buffersize, progress);
|
||||||
return new HdfsDataOutputStream(out, statistics, out.getInitialLen());
|
return createWrappedOutputStream(out, statistics, out.getInitialLen());
|
||||||
}
|
}
|
||||||
|
|
||||||
private DFSOutputStream append(String src, int buffersize, Progressable progress)
|
private DFSOutputStream append(String src, int buffersize, Progressable progress)
|
||||||
|
@ -2772,6 +2888,36 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void createEncryptionZone(String src, String keyName)
|
||||||
|
throws IOException {
|
||||||
|
checkOpen();
|
||||||
|
try {
|
||||||
|
namenode.createEncryptionZone(src, keyName);
|
||||||
|
} catch (RemoteException re) {
|
||||||
|
throw re.unwrapRemoteException(AccessControlException.class,
|
||||||
|
SafeModeException.class,
|
||||||
|
UnresolvedPathException.class);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public EncryptionZone getEZForPath(String src)
|
||||||
|
throws IOException {
|
||||||
|
checkOpen();
|
||||||
|
try {
|
||||||
|
final EncryptionZoneWithId ezi = namenode.getEZForPath(src);
|
||||||
|
return (ezi.getId() < 0) ? null : ezi;
|
||||||
|
} catch (RemoteException re) {
|
||||||
|
throw re.unwrapRemoteException(AccessControlException.class,
|
||||||
|
UnresolvedPathException.class);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public RemoteIterator<EncryptionZone> listEncryptionZones()
|
||||||
|
throws IOException {
|
||||||
|
checkOpen();
|
||||||
|
return new EncryptionZoneIterator(namenode);
|
||||||
|
}
|
||||||
|
|
||||||
public void setXAttr(String src, String name, byte[] value,
|
public void setXAttr(String src, String name, byte[] value,
|
||||||
EnumSet<XAttrSetFlag> flag) throws IOException {
|
EnumSet<XAttrSetFlag> flag) throws IOException {
|
||||||
checkOpen();
|
checkOpen();
|
||||||
|
|
|
@ -582,7 +582,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
|
||||||
public static final String DFS_TRUSTEDCHANNEL_RESOLVER_CLASS = "dfs.trustedchannel.resolver.class";
|
public static final String DFS_TRUSTEDCHANNEL_RESOLVER_CLASS = "dfs.trustedchannel.resolver.class";
|
||||||
public static final String DFS_DATA_TRANSFER_PROTECTION_KEY = "dfs.data.transfer.protection";
|
public static final String DFS_DATA_TRANSFER_PROTECTION_KEY = "dfs.data.transfer.protection";
|
||||||
public static final String DFS_DATA_TRANSFER_SASL_PROPS_RESOLVER_CLASS_KEY = "dfs.data.transfer.saslproperties.resolver.class";
|
public static final String DFS_DATA_TRANSFER_SASL_PROPS_RESOLVER_CLASS_KEY = "dfs.data.transfer.saslproperties.resolver.class";
|
||||||
|
public static final int DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES_DEFAULT = 100;
|
||||||
|
public static final String DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES = "dfs.namenode.list.encryption.zones.num.responses";
|
||||||
|
|
||||||
// Journal-node related configs. These are read on the JN side.
|
// Journal-node related configs. These are read on the JN side.
|
||||||
public static final String DFS_JOURNALNODE_EDITS_DIR_KEY = "dfs.journalnode.edits.dir";
|
public static final String DFS_JOURNALNODE_EDITS_DIR_KEY = "dfs.journalnode.edits.dir";
|
||||||
public static final String DFS_JOURNALNODE_EDITS_DIR_DEFAULT = "/tmp/hadoop/dfs/journalnode/";
|
public static final String DFS_JOURNALNODE_EDITS_DIR_DEFAULT = "/tmp/hadoop/dfs/journalnode/";
|
||||||
|
|
|
@ -56,6 +56,7 @@ import org.apache.hadoop.fs.UnresolvedLinkException;
|
||||||
import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
|
import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
|
||||||
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
|
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
|
||||||
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
|
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
|
||||||
|
import org.apache.hadoop.fs.FileEncryptionInfo;
|
||||||
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
|
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
|
||||||
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
|
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
|
||||||
import org.apache.hadoop.hdfs.protocol.datatransfer.InvalidEncryptionKeyException;
|
import org.apache.hadoop.hdfs.protocol.datatransfer.InvalidEncryptionKeyException;
|
||||||
|
@ -92,6 +93,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
private final boolean verifyChecksum;
|
private final boolean verifyChecksum;
|
||||||
private LocatedBlocks locatedBlocks = null;
|
private LocatedBlocks locatedBlocks = null;
|
||||||
private long lastBlockBeingWrittenLength = 0;
|
private long lastBlockBeingWrittenLength = 0;
|
||||||
|
private FileEncryptionInfo fileEncryptionInfo = null;
|
||||||
private DatanodeInfo currentNode = null;
|
private DatanodeInfo currentNode = null;
|
||||||
private LocatedBlock currentLocatedBlock = null;
|
private LocatedBlock currentLocatedBlock = null;
|
||||||
private long pos = 0;
|
private long pos = 0;
|
||||||
|
@ -301,6 +303,8 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fileEncryptionInfo = locatedBlocks.getFileEncryptionInfo();
|
||||||
|
|
||||||
currentNode = null;
|
currentNode = null;
|
||||||
return lastBlockBeingWrittenLength;
|
return lastBlockBeingWrittenLength;
|
||||||
}
|
}
|
||||||
|
@ -1525,6 +1529,10 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
return new ReadStatistics(readStatistics);
|
return new ReadStatistics(readStatistics);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public synchronized FileEncryptionInfo getFileEncryptionInfo() {
|
||||||
|
return fileEncryptionInfo;
|
||||||
|
}
|
||||||
|
|
||||||
private synchronized void closeCurrentBlockReader() {
|
private synchronized void closeCurrentBlockReader() {
|
||||||
if (blockReader == null) return;
|
if (blockReader == null) return;
|
||||||
// Close the current block reader so that the new caching settings can
|
// Close the current block reader so that the new caching settings can
|
||||||
|
|
|
@ -42,10 +42,12 @@ import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
import java.util.concurrent.atomic.AtomicReference;
|
import java.util.concurrent.atomic.AtomicReference;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
|
import org.apache.hadoop.crypto.CipherSuite;
|
||||||
import org.apache.hadoop.fs.CanSetDropBehind;
|
import org.apache.hadoop.fs.CanSetDropBehind;
|
||||||
import org.apache.hadoop.fs.CreateFlag;
|
import org.apache.hadoop.fs.CreateFlag;
|
||||||
import org.apache.hadoop.fs.FSOutputSummer;
|
import org.apache.hadoop.fs.FSOutputSummer;
|
||||||
import org.apache.hadoop.fs.FileAlreadyExistsException;
|
import org.apache.hadoop.fs.FileAlreadyExistsException;
|
||||||
|
import org.apache.hadoop.fs.FileEncryptionInfo;
|
||||||
import org.apache.hadoop.fs.ParentNotDirectoryException;
|
import org.apache.hadoop.fs.ParentNotDirectoryException;
|
||||||
import org.apache.hadoop.fs.Syncable;
|
import org.apache.hadoop.fs.Syncable;
|
||||||
import org.apache.hadoop.fs.permission.FsPermission;
|
import org.apache.hadoop.fs.permission.FsPermission;
|
||||||
|
@ -153,7 +155,8 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
private boolean shouldSyncBlock = false; // force blocks to disk upon close
|
private boolean shouldSyncBlock = false; // force blocks to disk upon close
|
||||||
private final AtomicReference<CachingStrategy> cachingStrategy;
|
private final AtomicReference<CachingStrategy> cachingStrategy;
|
||||||
private boolean failPacket = false;
|
private boolean failPacket = false;
|
||||||
|
private FileEncryptionInfo fileEncryptionInfo;
|
||||||
|
|
||||||
private static class Packet {
|
private static class Packet {
|
||||||
private static final long HEART_BEAT_SEQNO = -1L;
|
private static final long HEART_BEAT_SEQNO = -1L;
|
||||||
final long seqno; // sequencenumber of buffer in block
|
final long seqno; // sequencenumber of buffer in block
|
||||||
|
@ -1339,8 +1342,14 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
//
|
//
|
||||||
|
|
||||||
BlockConstructionStage bcs = recoveryFlag? stage.getRecoveryStage(): stage;
|
BlockConstructionStage bcs = recoveryFlag? stage.getRecoveryStage(): stage;
|
||||||
|
|
||||||
|
// We cannot change the block length in 'block' as it counts the number
|
||||||
|
// of bytes ack'ed.
|
||||||
|
ExtendedBlock blockCopy = new ExtendedBlock(block);
|
||||||
|
blockCopy.setNumBytes(blockSize);
|
||||||
|
|
||||||
// send the request
|
// send the request
|
||||||
new Sender(out).writeBlock(block, nodeStorageTypes[0], accessToken,
|
new Sender(out).writeBlock(blockCopy, nodeStorageTypes[0], accessToken,
|
||||||
dfsClient.clientName, nodes, nodeStorageTypes, null, bcs,
|
dfsClient.clientName, nodes, nodeStorageTypes, null, bcs,
|
||||||
nodes.length, block.getNumBytes(), bytesSent, newGS, checksum,
|
nodes.length, block.getNumBytes(), bytesSent, newGS, checksum,
|
||||||
cachingStrategy.get());
|
cachingStrategy.get());
|
||||||
|
@ -1560,6 +1569,7 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
this.fileId = stat.getFileId();
|
this.fileId = stat.getFileId();
|
||||||
this.blockSize = stat.getBlockSize();
|
this.blockSize = stat.getBlockSize();
|
||||||
this.blockReplication = stat.getReplication();
|
this.blockReplication = stat.getReplication();
|
||||||
|
this.fileEncryptionInfo = stat.getFileEncryptionInfo();
|
||||||
this.progress = progress;
|
this.progress = progress;
|
||||||
this.cachingStrategy = new AtomicReference<CachingStrategy>(
|
this.cachingStrategy = new AtomicReference<CachingStrategy>(
|
||||||
dfsClient.getDefaultWriteCachingStrategy());
|
dfsClient.getDefaultWriteCachingStrategy());
|
||||||
|
@ -1600,12 +1610,13 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
static DFSOutputStream newStreamForCreate(DFSClient dfsClient, String src,
|
static DFSOutputStream newStreamForCreate(DFSClient dfsClient, String src,
|
||||||
FsPermission masked, EnumSet<CreateFlag> flag, boolean createParent,
|
FsPermission masked, EnumSet<CreateFlag> flag, boolean createParent,
|
||||||
short replication, long blockSize, Progressable progress, int buffersize,
|
short replication, long blockSize, Progressable progress, int buffersize,
|
||||||
DataChecksum checksum, String[] favoredNodes) throws IOException {
|
DataChecksum checksum, String[] favoredNodes,
|
||||||
|
List<CipherSuite> cipherSuites) throws IOException {
|
||||||
final HdfsFileStatus stat;
|
final HdfsFileStatus stat;
|
||||||
try {
|
try {
|
||||||
stat = dfsClient.namenode.create(src, masked, dfsClient.clientName,
|
stat = dfsClient.namenode.create(src, masked, dfsClient.clientName,
|
||||||
new EnumSetWritable<CreateFlag>(flag), createParent, replication,
|
new EnumSetWritable<CreateFlag>(flag), createParent, replication,
|
||||||
blockSize);
|
blockSize, cipherSuites);
|
||||||
} catch(RemoteException re) {
|
} catch(RemoteException re) {
|
||||||
throw re.unwrapRemoteException(AccessControlException.class,
|
throw re.unwrapRemoteException(AccessControlException.class,
|
||||||
DSQuotaExceededException.class,
|
DSQuotaExceededException.class,
|
||||||
|
@ -1615,7 +1626,8 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
NSQuotaExceededException.class,
|
NSQuotaExceededException.class,
|
||||||
SafeModeException.class,
|
SafeModeException.class,
|
||||||
UnresolvedPathException.class,
|
UnresolvedPathException.class,
|
||||||
SnapshotAccessControlException.class);
|
SnapshotAccessControlException.class,
|
||||||
|
UnknownCipherSuiteException.class);
|
||||||
}
|
}
|
||||||
final DFSOutputStream out = new DFSOutputStream(dfsClient, src, stat,
|
final DFSOutputStream out = new DFSOutputStream(dfsClient, src, stat,
|
||||||
flag, progress, checksum, favoredNodes);
|
flag, progress, checksum, favoredNodes);
|
||||||
|
@ -1623,14 +1635,6 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
static DFSOutputStream newStreamForCreate(DFSClient dfsClient, String src,
|
|
||||||
FsPermission masked, EnumSet<CreateFlag> flag, boolean createParent,
|
|
||||||
short replication, long blockSize, Progressable progress, int buffersize,
|
|
||||||
DataChecksum checksum) throws IOException {
|
|
||||||
return newStreamForCreate(dfsClient, src, masked, flag, createParent, replication,
|
|
||||||
blockSize, progress, buffersize, checksum, null);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Construct a new output stream for append. */
|
/** Construct a new output stream for append. */
|
||||||
private DFSOutputStream(DFSClient dfsClient, String src,
|
private DFSOutputStream(DFSClient dfsClient, String src,
|
||||||
Progressable progress, LocatedBlock lastBlock, HdfsFileStatus stat,
|
Progressable progress, LocatedBlock lastBlock, HdfsFileStatus stat,
|
||||||
|
@ -1648,6 +1652,7 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
checksum.getBytesPerChecksum());
|
checksum.getBytesPerChecksum());
|
||||||
streamer = new DataStreamer();
|
streamer = new DataStreamer();
|
||||||
}
|
}
|
||||||
|
this.fileEncryptionInfo = stat.getFileEncryptionInfo();
|
||||||
}
|
}
|
||||||
|
|
||||||
static DFSOutputStream newStreamForAppend(DFSClient dfsClient, String src,
|
static DFSOutputStream newStreamForAppend(DFSClient dfsClient, String src,
|
||||||
|
@ -2172,10 +2177,17 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
/**
|
/**
|
||||||
* Returns the size of a file as it was when this stream was opened
|
* Returns the size of a file as it was when this stream was opened
|
||||||
*/
|
*/
|
||||||
long getInitialLen() {
|
public long getInitialLen() {
|
||||||
return initialFileSize;
|
return initialFileSize;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return the FileEncryptionInfo for this stream, or null if not encrypted.
|
||||||
|
*/
|
||||||
|
public FileEncryptionInfo getFileEncryptionInfo() {
|
||||||
|
return fileEncryptionInfo;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns the access token currently used by streamer, for testing only
|
* Returns the access token currently used by streamer, for testing only
|
||||||
*/
|
*/
|
||||||
|
|
|
@ -71,6 +71,9 @@ import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.hadoop.HadoopIllegalArgumentException;
|
import org.apache.hadoop.HadoopIllegalArgumentException;
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.crypto.key.KeyProvider;
|
||||||
|
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
|
||||||
|
import org.apache.hadoop.crypto.key.KeyProviderFactory;
|
||||||
import org.apache.hadoop.fs.BlockLocation;
|
import org.apache.hadoop.fs.BlockLocation;
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
|
@ -1722,4 +1725,39 @@ public class DFSUtil {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a new KeyProviderCryptoExtension by wrapping the
|
||||||
|
* KeyProvider specified in the given Configuration.
|
||||||
|
*
|
||||||
|
* @param conf Configuration specifying a single, non-transient KeyProvider.
|
||||||
|
* @return new KeyProviderCryptoExtension, or null if no provider was found.
|
||||||
|
* @throws IOException if the KeyProvider is improperly specified in
|
||||||
|
* the Configuration
|
||||||
|
*/
|
||||||
|
public static KeyProviderCryptoExtension createKeyProviderCryptoExtension(
|
||||||
|
final Configuration conf) throws IOException {
|
||||||
|
final List<KeyProvider> providers = KeyProviderFactory.getProviders(conf);
|
||||||
|
if (providers == null || providers.size() == 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (providers.size() > 1) {
|
||||||
|
StringBuilder builder = new StringBuilder();
|
||||||
|
builder.append("Found multiple KeyProviders but only one is permitted [");
|
||||||
|
String prefix = " ";
|
||||||
|
for (KeyProvider kp: providers) {
|
||||||
|
builder.append(prefix + kp.toString());
|
||||||
|
prefix = ", ";
|
||||||
|
}
|
||||||
|
builder.append("]");
|
||||||
|
throw new IOException(builder.toString());
|
||||||
|
}
|
||||||
|
KeyProviderCryptoExtension provider = KeyProviderCryptoExtension
|
||||||
|
.createKeyProviderCryptoExtension(providers.get(0));
|
||||||
|
if (provider.isTransient()) {
|
||||||
|
throw new IOException("KeyProvider " + provider.toString()
|
||||||
|
+ " was found but it is a transient provider.");
|
||||||
|
}
|
||||||
|
return provider;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -61,7 +61,6 @@ import org.apache.hadoop.fs.permission.AclStatus;
|
||||||
import org.apache.hadoop.fs.permission.FsPermission;
|
import org.apache.hadoop.fs.permission.FsPermission;
|
||||||
import org.apache.hadoop.fs.permission.FsAction;
|
import org.apache.hadoop.fs.permission.FsAction;
|
||||||
import org.apache.hadoop.hdfs.client.HdfsAdmin;
|
import org.apache.hadoop.hdfs.client.HdfsAdmin;
|
||||||
import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
|
|
||||||
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
|
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
|
||||||
import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
|
import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
|
||||||
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
|
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
|
||||||
|
@ -69,6 +68,7 @@ import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
|
||||||
import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
|
import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
|
||||||
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
|
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
|
||||||
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
|
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
|
||||||
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
|
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
|
||||||
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
|
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
|
||||||
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
|
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
|
||||||
|
@ -291,8 +291,9 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
@Override
|
@Override
|
||||||
public FSDataInputStream doCall(final Path p)
|
public FSDataInputStream doCall(final Path p)
|
||||||
throws IOException, UnresolvedLinkException {
|
throws IOException, UnresolvedLinkException {
|
||||||
return new HdfsDataInputStream(
|
final DFSInputStream dfsis =
|
||||||
dfs.open(getPathName(p), bufferSize, verifyChecksum));
|
dfs.open(getPathName(p), bufferSize, verifyChecksum);
|
||||||
|
return dfs.createWrappedInputStream(dfsis);
|
||||||
}
|
}
|
||||||
@Override
|
@Override
|
||||||
public FSDataInputStream next(final FileSystem fs, final Path p)
|
public FSDataInputStream next(final FileSystem fs, final Path p)
|
||||||
|
@ -357,7 +358,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
: EnumSet.of(CreateFlag.CREATE),
|
: EnumSet.of(CreateFlag.CREATE),
|
||||||
true, replication, blockSize, progress, bufferSize, null,
|
true, replication, blockSize, progress, bufferSize, null,
|
||||||
favoredNodes);
|
favoredNodes);
|
||||||
return new HdfsDataOutputStream(out, statistics);
|
return dfs.createWrappedOutputStream(out, statistics);
|
||||||
}
|
}
|
||||||
@Override
|
@Override
|
||||||
public HdfsDataOutputStream next(final FileSystem fs, final Path p)
|
public HdfsDataOutputStream next(final FileSystem fs, final Path p)
|
||||||
|
@ -385,9 +386,10 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
@Override
|
@Override
|
||||||
public FSDataOutputStream doCall(final Path p)
|
public FSDataOutputStream doCall(final Path p)
|
||||||
throws IOException, UnresolvedLinkException {
|
throws IOException, UnresolvedLinkException {
|
||||||
return new HdfsDataOutputStream(dfs.create(getPathName(p), permission,
|
final DFSOutputStream dfsos = dfs.create(getPathName(p), permission,
|
||||||
cflags, replication, blockSize, progress, bufferSize, checksumOpt),
|
cflags, replication, blockSize, progress, bufferSize,
|
||||||
statistics);
|
checksumOpt);
|
||||||
|
return dfs.createWrappedOutputStream(dfsos, statistics);
|
||||||
}
|
}
|
||||||
@Override
|
@Override
|
||||||
public FSDataOutputStream next(final FileSystem fs, final Path p)
|
public FSDataOutputStream next(final FileSystem fs, final Path p)
|
||||||
|
@ -404,11 +406,12 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
short replication, long blockSize, Progressable progress,
|
short replication, long blockSize, Progressable progress,
|
||||||
ChecksumOpt checksumOpt) throws IOException {
|
ChecksumOpt checksumOpt) throws IOException {
|
||||||
statistics.incrementWriteOps(1);
|
statistics.incrementWriteOps(1);
|
||||||
return new HdfsDataOutputStream(dfs.primitiveCreate(
|
final DFSOutputStream dfsos = dfs.primitiveCreate(
|
||||||
getPathName(fixRelativePart(f)),
|
getPathName(fixRelativePart(f)),
|
||||||
absolutePermission, flag, true, replication, blockSize,
|
absolutePermission, flag, true, replication, blockSize,
|
||||||
progress, bufferSize, checksumOpt),statistics);
|
progress, bufferSize, checksumOpt);
|
||||||
}
|
return dfs.createWrappedOutputStream(dfsos, statistics);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Same as create(), except fails if parent directory doesn't already exist.
|
* Same as create(), except fails if parent directory doesn't already exist.
|
||||||
|
@ -428,9 +431,9 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
@Override
|
@Override
|
||||||
public FSDataOutputStream doCall(final Path p) throws IOException,
|
public FSDataOutputStream doCall(final Path p) throws IOException,
|
||||||
UnresolvedLinkException {
|
UnresolvedLinkException {
|
||||||
return new HdfsDataOutputStream(dfs.create(getPathName(p), permission,
|
final DFSOutputStream dfsos = dfs.create(getPathName(p), permission,
|
||||||
flag, false, replication, blockSize, progress, bufferSize, null),
|
flag, false, replication, blockSize, progress, bufferSize, null);
|
||||||
statistics);
|
return dfs.createWrappedOutputStream(dfsos, statistics);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -1822,6 +1825,25 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
}.resolve(this, absF);
|
}.resolve(this, absF);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* HDFS only */
|
||||||
|
public void createEncryptionZone(Path path, String keyName)
|
||||||
|
throws IOException {
|
||||||
|
dfs.createEncryptionZone(getPathName(path), keyName);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* HDFS only */
|
||||||
|
public EncryptionZone getEZForPath(Path path)
|
||||||
|
throws IOException {
|
||||||
|
Preconditions.checkNotNull(path);
|
||||||
|
return dfs.getEZForPath(getPathName(path));
|
||||||
|
}
|
||||||
|
|
||||||
|
/* HDFS only */
|
||||||
|
public RemoteIterator<EncryptionZone> listEncryptionZones()
|
||||||
|
throws IOException {
|
||||||
|
return dfs.listEncryptionZones();
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void setXAttr(Path path, final String name, final byte[] value,
|
public void setXAttr(Path path, final String name, final byte[] value,
|
||||||
final EnumSet<XAttrSetFlag> flag) throws IOException {
|
final EnumSet<XAttrSetFlag> flag) throws IOException {
|
||||||
|
|
|
@ -0,0 +1,38 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.hdfs;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
|
|
||||||
|
@InterfaceAudience.Public
|
||||||
|
@InterfaceStability.Evolving
|
||||||
|
public class UnknownCipherSuiteException extends IOException {
|
||||||
|
private static final long serialVersionUID = 8957192l;
|
||||||
|
|
||||||
|
public UnknownCipherSuiteException() {
|
||||||
|
super();
|
||||||
|
}
|
||||||
|
|
||||||
|
public UnknownCipherSuiteException(String msg) {
|
||||||
|
super(msg);
|
||||||
|
}
|
||||||
|
}
|
|
@ -49,9 +49,9 @@ public class XAttrHelper {
|
||||||
Preconditions.checkNotNull(name, "XAttr name cannot be null.");
|
Preconditions.checkNotNull(name, "XAttr name cannot be null.");
|
||||||
|
|
||||||
final int prefixIndex = name.indexOf(".");
|
final int prefixIndex = name.indexOf(".");
|
||||||
if (prefixIndex < 4) {// Prefix length is at least 4.
|
if (prefixIndex < 3) {// Prefix length is at least 3.
|
||||||
throw new HadoopIllegalArgumentException("An XAttr name must be " +
|
throw new HadoopIllegalArgumentException("An XAttr name must be " +
|
||||||
"prefixed with user/trusted/security/system, followed by a '.'");
|
"prefixed with user/trusted/security/system/raw, followed by a '.'");
|
||||||
} else if (prefixIndex == name.length() - 1) {
|
} else if (prefixIndex == name.length() - 1) {
|
||||||
throw new HadoopIllegalArgumentException("XAttr name cannot be empty.");
|
throw new HadoopIllegalArgumentException("XAttr name cannot be empty.");
|
||||||
}
|
}
|
||||||
|
@ -66,9 +66,11 @@ public class XAttrHelper {
|
||||||
ns = NameSpace.SYSTEM;
|
ns = NameSpace.SYSTEM;
|
||||||
} else if (prefix.equals(NameSpace.SECURITY.toString().toLowerCase())) {
|
} else if (prefix.equals(NameSpace.SECURITY.toString().toLowerCase())) {
|
||||||
ns = NameSpace.SECURITY;
|
ns = NameSpace.SECURITY;
|
||||||
|
} else if (prefix.equals(NameSpace.RAW.toString().toLowerCase())) {
|
||||||
|
ns = NameSpace.RAW;
|
||||||
} else {
|
} else {
|
||||||
throw new HadoopIllegalArgumentException("An XAttr name must be " +
|
throw new HadoopIllegalArgumentException("An XAttr name must be " +
|
||||||
"prefixed with user/trusted/security/system, followed by a '.'");
|
"prefixed with user/trusted/security/system/raw, followed by a '.'");
|
||||||
}
|
}
|
||||||
XAttr xAttr = (new XAttr.Builder()).setNameSpace(ns).setName(name.
|
XAttr xAttr = (new XAttr.Builder()).setNameSpace(ns).setName(name.
|
||||||
substring(prefixIndex + 1)).setValue(value).build();
|
substring(prefixIndex + 1)).setValue(value).build();
|
||||||
|
|
|
@ -17,6 +17,7 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.hadoop.hdfs.client;
|
package org.apache.hadoop.hdfs.client;
|
||||||
|
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.util.EnumSet;
|
import java.util.EnumSet;
|
||||||
|
@ -33,7 +34,9 @@ import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
|
||||||
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
|
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
|
||||||
import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
|
import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
|
||||||
import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
|
import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
|
||||||
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
|
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
|
||||||
|
import org.apache.hadoop.security.AccessControlException;
|
||||||
import org.apache.hadoop.hdfs.tools.DFSAdmin;
|
import org.apache.hadoop.hdfs.tools.DFSAdmin;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -225,4 +228,51 @@ public class HdfsAdmin {
|
||||||
public RemoteIterator<CachePoolEntry> listCachePools() throws IOException {
|
public RemoteIterator<CachePoolEntry> listCachePools() throws IOException {
|
||||||
return dfs.listCachePools();
|
return dfs.listCachePools();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create an encryption zone rooted at an empty existing directory, using the
|
||||||
|
* specified encryption key. An encryption zone has an associated encryption
|
||||||
|
* key used when reading and writing files within the zone.
|
||||||
|
*
|
||||||
|
* @param path The path of the root of the encryption zone. Must refer to
|
||||||
|
* an empty, existing directory.
|
||||||
|
* @param keyName Name of key available at the KeyProvider.
|
||||||
|
* @throws IOException if there was a general IO exception
|
||||||
|
* @throws AccessControlException if the caller does not have access to path
|
||||||
|
* @throws FileNotFoundException if the path does not exist
|
||||||
|
*/
|
||||||
|
public void createEncryptionZone(Path path, String keyName)
|
||||||
|
throws IOException, AccessControlException, FileNotFoundException {
|
||||||
|
dfs.createEncryptionZone(path, keyName);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the path of the encryption zone for a given file or directory.
|
||||||
|
*
|
||||||
|
* @param path The path to get the ez for.
|
||||||
|
*
|
||||||
|
* @return The EncryptionZone of the ez, or null if path is not in an ez.
|
||||||
|
* @throws IOException if there was a general IO exception
|
||||||
|
* @throws AccessControlException if the caller does not have access to path
|
||||||
|
* @throws FileNotFoundException if the path does not exist
|
||||||
|
*/
|
||||||
|
public EncryptionZone getEncryptionZoneForPath(Path path)
|
||||||
|
throws IOException, AccessControlException, FileNotFoundException {
|
||||||
|
return dfs.getEZForPath(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns a RemoteIterator which can be used to list the encryption zones
|
||||||
|
* in HDFS. For large numbers of encryption zones, the iterator will fetch
|
||||||
|
* the list of zones in a number of small batches.
|
||||||
|
* <p/>
|
||||||
|
* Since the list is fetched in batches, it does not represent a
|
||||||
|
* consistent snapshot of the entire list of encryption zones.
|
||||||
|
* <p/>
|
||||||
|
* This method can only be called by HDFS superusers.
|
||||||
|
*/
|
||||||
|
public RemoteIterator<EncryptionZone> listEncryptionZones()
|
||||||
|
throws IOException {
|
||||||
|
return dfs.listEncryptionZones();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -17,17 +17,21 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.hadoop.hdfs.client;
|
package org.apache.hadoop.hdfs.client;
|
||||||
|
|
||||||
|
import java.io.InputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
import org.apache.hadoop.fs.FSDataInputStream;
|
import org.apache.hadoop.fs.FSDataInputStream;
|
||||||
|
import org.apache.hadoop.crypto.CryptoInputStream;
|
||||||
import org.apache.hadoop.hdfs.DFSInputStream;
|
import org.apache.hadoop.hdfs.DFSInputStream;
|
||||||
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
|
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
|
||||||
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
|
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
|
||||||
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
|
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
|
||||||
|
|
||||||
|
import com.google.common.base.Preconditions;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The Hdfs implementation of {@link FSDataInputStream}.
|
* The Hdfs implementation of {@link FSDataInputStream}.
|
||||||
*/
|
*/
|
||||||
|
@ -38,25 +42,49 @@ public class HdfsDataInputStream extends FSDataInputStream {
|
||||||
super(in);
|
super(in);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public HdfsDataInputStream(CryptoInputStream in) throws IOException {
|
||||||
|
super(in);
|
||||||
|
Preconditions.checkArgument(in.getWrappedStream() instanceof DFSInputStream,
|
||||||
|
"CryptoInputStream should wrap a DFSInputStream");
|
||||||
|
}
|
||||||
|
|
||||||
|
private DFSInputStream getDFSInputStream() {
|
||||||
|
if (in instanceof CryptoInputStream) {
|
||||||
|
return (DFSInputStream) ((CryptoInputStream) in).getWrappedStream();
|
||||||
|
}
|
||||||
|
return (DFSInputStream) in;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get a reference to the wrapped output stream. We always want to return the
|
||||||
|
* actual underlying InputStream, even when we're using a CryptoStream. e.g.
|
||||||
|
* in the delegated methods below.
|
||||||
|
*
|
||||||
|
* @return the underlying output stream
|
||||||
|
*/
|
||||||
|
public InputStream getWrappedStream() {
|
||||||
|
return in;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get the datanode from which the stream is currently reading.
|
* Get the datanode from which the stream is currently reading.
|
||||||
*/
|
*/
|
||||||
public DatanodeInfo getCurrentDatanode() {
|
public DatanodeInfo getCurrentDatanode() {
|
||||||
return ((DFSInputStream) in).getCurrentDatanode();
|
return getDFSInputStream().getCurrentDatanode();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get the block containing the target position.
|
* Get the block containing the target position.
|
||||||
*/
|
*/
|
||||||
public ExtendedBlock getCurrentBlock() {
|
public ExtendedBlock getCurrentBlock() {
|
||||||
return ((DFSInputStream) in).getCurrentBlock();
|
return getDFSInputStream().getCurrentBlock();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get the collection of blocks that has already been located.
|
* Get the collection of blocks that has already been located.
|
||||||
*/
|
*/
|
||||||
public synchronized List<LocatedBlock> getAllBlocks() throws IOException {
|
public synchronized List<LocatedBlock> getAllBlocks() throws IOException {
|
||||||
return ((DFSInputStream) in).getAllBlocks();
|
return getDFSInputStream().getAllBlocks();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -66,7 +94,7 @@ public class HdfsDataInputStream extends FSDataInputStream {
|
||||||
* @return The visible length of the file.
|
* @return The visible length of the file.
|
||||||
*/
|
*/
|
||||||
public long getVisibleLength() throws IOException {
|
public long getVisibleLength() throws IOException {
|
||||||
return ((DFSInputStream) in).getFileLength();
|
return getDFSInputStream().getFileLength();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -76,6 +104,6 @@ public class HdfsDataInputStream extends FSDataInputStream {
|
||||||
* bytes read through HdfsDataInputStream.
|
* bytes read through HdfsDataInputStream.
|
||||||
*/
|
*/
|
||||||
public synchronized DFSInputStream.ReadStatistics getReadStatistics() {
|
public synchronized DFSInputStream.ReadStatistics getReadStatistics() {
|
||||||
return ((DFSInputStream) in).getReadStatistics();
|
return getDFSInputStream().getReadStatistics();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,14 +18,18 @@
|
||||||
package org.apache.hadoop.hdfs.client;
|
package org.apache.hadoop.hdfs.client;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.io.OutputStream;
|
||||||
import java.util.EnumSet;
|
import java.util.EnumSet;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
|
import org.apache.hadoop.crypto.CryptoOutputStream;
|
||||||
import org.apache.hadoop.fs.FSDataOutputStream;
|
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
import org.apache.hadoop.hdfs.DFSOutputStream;
|
import org.apache.hadoop.hdfs.DFSOutputStream;
|
||||||
|
|
||||||
|
import com.google.common.base.Preconditions;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The Hdfs implementation of {@link FSDataOutputStream}.
|
* The Hdfs implementation of {@link FSDataOutputStream}.
|
||||||
*/
|
*/
|
||||||
|
@ -42,6 +46,18 @@ public class HdfsDataOutputStream extends FSDataOutputStream {
|
||||||
this(out, stats, 0L);
|
this(out, stats, 0L);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public HdfsDataOutputStream(CryptoOutputStream out, FileSystem.Statistics stats,
|
||||||
|
long startPosition) throws IOException {
|
||||||
|
super(out, stats, startPosition);
|
||||||
|
Preconditions.checkArgument(out.getWrappedStream() instanceof DFSOutputStream,
|
||||||
|
"CryptoOutputStream should wrap a DFSOutputStream");
|
||||||
|
}
|
||||||
|
|
||||||
|
public HdfsDataOutputStream(CryptoOutputStream out, FileSystem.Statistics stats)
|
||||||
|
throws IOException {
|
||||||
|
this(out, stats, 0L);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get the actual number of replicas of the current block.
|
* Get the actual number of replicas of the current block.
|
||||||
*
|
*
|
||||||
|
@ -55,7 +71,11 @@ public class HdfsDataOutputStream extends FSDataOutputStream {
|
||||||
* @return the number of valid replicas of the current block
|
* @return the number of valid replicas of the current block
|
||||||
*/
|
*/
|
||||||
public synchronized int getCurrentBlockReplication() throws IOException {
|
public synchronized int getCurrentBlockReplication() throws IOException {
|
||||||
return ((DFSOutputStream)getWrappedStream()).getCurrentBlockReplication();
|
OutputStream wrappedStream = getWrappedStream();
|
||||||
|
if (wrappedStream instanceof CryptoOutputStream) {
|
||||||
|
wrappedStream = ((CryptoOutputStream) wrappedStream).getWrappedStream();
|
||||||
|
}
|
||||||
|
return ((DFSOutputStream) wrappedStream).getCurrentBlockReplication();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -67,14 +87,20 @@ public class HdfsDataOutputStream extends FSDataOutputStream {
|
||||||
* @see FSDataOutputStream#hsync()
|
* @see FSDataOutputStream#hsync()
|
||||||
*/
|
*/
|
||||||
public void hsync(EnumSet<SyncFlag> syncFlags) throws IOException {
|
public void hsync(EnumSet<SyncFlag> syncFlags) throws IOException {
|
||||||
((DFSOutputStream) getWrappedStream()).hsync(syncFlags);
|
OutputStream wrappedStream = getWrappedStream();
|
||||||
|
if (wrappedStream instanceof CryptoOutputStream) {
|
||||||
|
((CryptoOutputStream) wrappedStream).flush();
|
||||||
|
wrappedStream = ((CryptoOutputStream) wrappedStream).getWrappedStream();
|
||||||
|
}
|
||||||
|
((DFSOutputStream) wrappedStream).hsync(syncFlags);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static enum SyncFlag {
|
public static enum SyncFlag {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* When doing sync to DataNodes, also update the metadata (block
|
* When doing sync to DataNodes, also update the metadata (block length) in
|
||||||
* length) in the NameNode
|
* the NameNode.
|
||||||
*/
|
*/
|
||||||
UPDATE_LENGTH;
|
UPDATE_LENGTH;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -24,6 +24,7 @@ import java.util.List;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
|
import org.apache.hadoop.crypto.CipherSuite;
|
||||||
import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries;
|
import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries;
|
||||||
import org.apache.hadoop.fs.CacheFlag;
|
import org.apache.hadoop.fs.CacheFlag;
|
||||||
import org.apache.hadoop.fs.ContentSummary;
|
import org.apache.hadoop.fs.ContentSummary;
|
||||||
|
@ -188,7 +189,8 @@ public interface ClientProtocol {
|
||||||
@AtMostOnce
|
@AtMostOnce
|
||||||
public HdfsFileStatus create(String src, FsPermission masked,
|
public HdfsFileStatus create(String src, FsPermission masked,
|
||||||
String clientName, EnumSetWritable<CreateFlag> flag,
|
String clientName, EnumSetWritable<CreateFlag> flag,
|
||||||
boolean createParent, short replication, long blockSize)
|
boolean createParent, short replication, long blockSize,
|
||||||
|
List<CipherSuite> cipherSuites)
|
||||||
throws AccessControlException, AlreadyBeingCreatedException,
|
throws AccessControlException, AlreadyBeingCreatedException,
|
||||||
DSQuotaExceededException, FileAlreadyExistsException,
|
DSQuotaExceededException, FileAlreadyExistsException,
|
||||||
FileNotFoundException, NSQuotaExceededException,
|
FileNotFoundException, NSQuotaExceededException,
|
||||||
|
@ -1280,6 +1282,31 @@ public interface ClientProtocol {
|
||||||
@Idempotent
|
@Idempotent
|
||||||
public AclStatus getAclStatus(String src) throws IOException;
|
public AclStatus getAclStatus(String src) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create an encryption zone
|
||||||
|
*/
|
||||||
|
@AtMostOnce
|
||||||
|
public void createEncryptionZone(String src, String keyName)
|
||||||
|
throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the encryption zone for a path.
|
||||||
|
*/
|
||||||
|
@Idempotent
|
||||||
|
public EncryptionZoneWithId getEZForPath(String src)
|
||||||
|
throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Used to implement cursor-based batched listing of {@EncryptionZone}s.
|
||||||
|
*
|
||||||
|
* @param prevId ID of the last item in the previous batch. If there is no
|
||||||
|
* previous batch, a negative value can be used.
|
||||||
|
* @return Batch of encryption zones.
|
||||||
|
*/
|
||||||
|
@Idempotent
|
||||||
|
public BatchedEntries<EncryptionZoneWithId> listEncryptionZones(
|
||||||
|
long prevId) throws IOException;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Set xattr of a file or directory.
|
* Set xattr of a file or directory.
|
||||||
* The name must be prefixed with the namespace followed by ".". For example,
|
* The name must be prefixed with the namespace followed by ".". For example,
|
||||||
|
@ -1321,7 +1348,6 @@ public interface ClientProtocol {
|
||||||
* Refer to the HDFS extended attributes user documentation for details.
|
* Refer to the HDFS extended attributes user documentation for details.
|
||||||
*
|
*
|
||||||
* @param src file or directory
|
* @param src file or directory
|
||||||
* @param xAttrs xAttrs to get
|
|
||||||
* @return List<XAttr> <code>XAttr</code> list
|
* @return List<XAttr> <code>XAttr</code> list
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
|
|
|
@ -0,0 +1,79 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.hdfs.protocol;
|
||||||
|
|
||||||
|
import org.apache.commons.lang.builder.EqualsBuilder;
|
||||||
|
import org.apache.commons.lang.builder.HashCodeBuilder;
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A simple class for representing an encryption zone. Presently an encryption
|
||||||
|
* zone only has a path (the root of the encryption zone) and a key name.
|
||||||
|
*/
|
||||||
|
@InterfaceAudience.Public
|
||||||
|
@InterfaceStability.Evolving
|
||||||
|
public class EncryptionZone {
|
||||||
|
|
||||||
|
private final String path;
|
||||||
|
private final String keyName;
|
||||||
|
|
||||||
|
public EncryptionZone(String path, String keyName) {
|
||||||
|
this.path = path;
|
||||||
|
this.keyName = keyName;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getPath() {
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getKeyName() {
|
||||||
|
return keyName;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int hashCode() {
|
||||||
|
return new HashCodeBuilder(13, 31).
|
||||||
|
append(path).append(keyName).
|
||||||
|
toHashCode();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean equals(Object obj) {
|
||||||
|
if (obj == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (obj == this) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (obj.getClass() != getClass()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
EncryptionZone rhs = (EncryptionZone) obj;
|
||||||
|
return new EqualsBuilder().
|
||||||
|
append(path, rhs.path).
|
||||||
|
append(keyName, rhs.keyName).
|
||||||
|
isEquals();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
return "EncryptionZone [path=" + path + ", keyName=" + keyName + "]";
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,51 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.hdfs.protocol;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
|
import org.apache.hadoop.fs.RemoteIterator;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* EncryptionZoneIterator is a remote iterator that iterates over encryption
|
||||||
|
* zones. It supports retrying in case of namenode failover.
|
||||||
|
*/
|
||||||
|
@InterfaceAudience.Private
|
||||||
|
@InterfaceStability.Evolving
|
||||||
|
public class EncryptionZoneIterator implements RemoteIterator<EncryptionZone> {
|
||||||
|
|
||||||
|
private final EncryptionZoneWithIdIterator iterator;
|
||||||
|
|
||||||
|
public EncryptionZoneIterator(ClientProtocol namenode) {
|
||||||
|
iterator = new EncryptionZoneWithIdIterator(namenode);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean hasNext() throws IOException {
|
||||||
|
return iterator.hasNext();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public EncryptionZone next() throws IOException {
|
||||||
|
EncryptionZoneWithId ezwi = iterator.next();
|
||||||
|
return ezwi.toEncryptionZone();
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,64 @@
|
||||||
|
package org.apache.hadoop.hdfs.protocol;
|
||||||
|
|
||||||
|
import org.apache.commons.lang.builder.HashCodeBuilder;
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Internal class similar to an {@link EncryptionZone} which also holds a
|
||||||
|
* unique id. Used to implement batched listing of encryption zones.
|
||||||
|
*/
|
||||||
|
@InterfaceAudience.Private
|
||||||
|
public class EncryptionZoneWithId extends EncryptionZone {
|
||||||
|
|
||||||
|
final long id;
|
||||||
|
|
||||||
|
public EncryptionZoneWithId(String path, String keyName, long id) {
|
||||||
|
super(path, keyName);
|
||||||
|
this.id = id;
|
||||||
|
}
|
||||||
|
|
||||||
|
public long getId() {
|
||||||
|
return id;
|
||||||
|
}
|
||||||
|
|
||||||
|
EncryptionZone toEncryptionZone() {
|
||||||
|
return new EncryptionZone(getPath(), getKeyName());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int hashCode() {
|
||||||
|
return new HashCodeBuilder(17, 29)
|
||||||
|
.append(super.hashCode())
|
||||||
|
.append(id)
|
||||||
|
.toHashCode();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean equals(Object o) {
|
||||||
|
if (this == o) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (o == null || getClass() != o.getClass()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (!super.equals(o)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
EncryptionZoneWithId that = (EncryptionZoneWithId) o;
|
||||||
|
|
||||||
|
if (id != that.id) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
return "EncryptionZoneWithId [" +
|
||||||
|
"id=" + id +
|
||||||
|
", " + super.toString() +
|
||||||
|
']';
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,53 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.hdfs.protocol;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
|
import org.apache.hadoop.fs.BatchedRemoteIterator;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Used on the client-side to iterate over the list of encryption zones
|
||||||
|
* stored on the namenode.
|
||||||
|
*/
|
||||||
|
@InterfaceAudience.Private
|
||||||
|
@InterfaceStability.Evolving
|
||||||
|
public class EncryptionZoneWithIdIterator
|
||||||
|
extends BatchedRemoteIterator<Long, EncryptionZoneWithId> {
|
||||||
|
|
||||||
|
private final ClientProtocol namenode;
|
||||||
|
|
||||||
|
EncryptionZoneWithIdIterator(ClientProtocol namenode) {
|
||||||
|
super(Long.valueOf(0));
|
||||||
|
this.namenode = namenode;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BatchedEntries<EncryptionZoneWithId> makeRequest(Long prevId)
|
||||||
|
throws IOException {
|
||||||
|
return namenode.listEncryptionZones(prevId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Long elementToPrevKey(EncryptionZoneWithId entry) {
|
||||||
|
return entry.getId();
|
||||||
|
}
|
||||||
|
}
|
|
@ -21,6 +21,7 @@ import java.net.URI;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
|
import org.apache.hadoop.fs.FileEncryptionInfo;
|
||||||
import org.apache.hadoop.fs.FileStatus;
|
import org.apache.hadoop.fs.FileStatus;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.fs.permission.FsPermission;
|
import org.apache.hadoop.fs.permission.FsPermission;
|
||||||
|
@ -45,6 +46,8 @@ public class HdfsFileStatus {
|
||||||
private final String group;
|
private final String group;
|
||||||
private final long fileId;
|
private final long fileId;
|
||||||
|
|
||||||
|
private final FileEncryptionInfo feInfo;
|
||||||
|
|
||||||
// Used by dir, not including dot and dotdot. Always zero for a regular file.
|
// Used by dir, not including dot and dotdot. Always zero for a regular file.
|
||||||
private final int childrenNum;
|
private final int childrenNum;
|
||||||
private final byte storagePolicy;
|
private final byte storagePolicy;
|
||||||
|
@ -64,11 +67,13 @@ public class HdfsFileStatus {
|
||||||
* @param group the group of the path
|
* @param group the group of the path
|
||||||
* @param path the local name in java UTF8 encoding the same as that in-memory
|
* @param path the local name in java UTF8 encoding the same as that in-memory
|
||||||
* @param fileId the file id
|
* @param fileId the file id
|
||||||
|
* @param feInfo the file's encryption info
|
||||||
*/
|
*/
|
||||||
public HdfsFileStatus(long length, boolean isdir, int block_replication,
|
public HdfsFileStatus(long length, boolean isdir, int block_replication,
|
||||||
long blocksize, long modification_time, long access_time,
|
long blocksize, long modification_time, long access_time,
|
||||||
FsPermission permission, String owner, String group, byte[] symlink,
|
FsPermission permission, String owner, String group, byte[] symlink,
|
||||||
byte[] path, long fileId, int childrenNum, byte storagePolicy) {
|
byte[] path, long fileId, int childrenNum, FileEncryptionInfo feInfo,
|
||||||
|
byte storagePolicy) {
|
||||||
this.length = length;
|
this.length = length;
|
||||||
this.isdir = isdir;
|
this.isdir = isdir;
|
||||||
this.block_replication = (short)block_replication;
|
this.block_replication = (short)block_replication;
|
||||||
|
@ -86,6 +91,7 @@ public class HdfsFileStatus {
|
||||||
this.path = path;
|
this.path = path;
|
||||||
this.fileId = fileId;
|
this.fileId = fileId;
|
||||||
this.childrenNum = childrenNum;
|
this.childrenNum = childrenNum;
|
||||||
|
this.feInfo = feInfo;
|
||||||
this.storagePolicy = storagePolicy;
|
this.storagePolicy = storagePolicy;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -240,6 +246,10 @@ public class HdfsFileStatus {
|
||||||
return fileId;
|
return fileId;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public final FileEncryptionInfo getFileEncryptionInfo() {
|
||||||
|
return feInfo;
|
||||||
|
}
|
||||||
|
|
||||||
public final int getChildrenNum() {
|
public final int getChildrenNum() {
|
||||||
return childrenNum;
|
return childrenNum;
|
||||||
}
|
}
|
||||||
|
|
|
@ -21,6 +21,7 @@ import java.net.URI;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
|
import org.apache.hadoop.fs.FileEncryptionInfo;
|
||||||
import org.apache.hadoop.fs.LocatedFileStatus;
|
import org.apache.hadoop.fs.LocatedFileStatus;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.fs.permission.FsPermission;
|
import org.apache.hadoop.fs.permission.FsPermission;
|
||||||
|
@ -51,15 +52,16 @@ public class HdfsLocatedFileStatus extends HdfsFileStatus {
|
||||||
* @param path local path name in java UTF8 format
|
* @param path local path name in java UTF8 format
|
||||||
* @param fileId the file id
|
* @param fileId the file id
|
||||||
* @param locations block locations
|
* @param locations block locations
|
||||||
|
* @param feInfo file encryption info
|
||||||
*/
|
*/
|
||||||
public HdfsLocatedFileStatus(long length, boolean isdir,
|
public HdfsLocatedFileStatus(long length, boolean isdir,
|
||||||
int block_replication, long blocksize, long modification_time,
|
int block_replication, long blocksize, long modification_time,
|
||||||
long access_time, FsPermission permission, String owner, String group,
|
long access_time, FsPermission permission, String owner, String group,
|
||||||
byte[] symlink, byte[] path, long fileId, LocatedBlocks locations,
|
byte[] symlink, byte[] path, long fileId, LocatedBlocks locations,
|
||||||
int childrenNum, byte storagePolicy) {
|
int childrenNum, FileEncryptionInfo feInfo, byte storagePolicy) {
|
||||||
super(length, isdir, block_replication, blocksize, modification_time,
|
super(length, isdir, block_replication, blocksize, modification_time,
|
||||||
access_time, permission, owner, group, symlink, path, fileId,
|
access_time, permission, owner, group, symlink, path, fileId,
|
||||||
childrenNum, storagePolicy);
|
childrenNum, feInfo, storagePolicy);
|
||||||
this.locations = locations;
|
this.locations = locations;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -23,6 +23,7 @@ import java.util.Comparator;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
|
import org.apache.hadoop.fs.FileEncryptionInfo;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Collection of blocks with their locations and the file length.
|
* Collection of blocks with their locations and the file length.
|
||||||
|
@ -35,22 +36,23 @@ public class LocatedBlocks {
|
||||||
private final boolean underConstruction;
|
private final boolean underConstruction;
|
||||||
private LocatedBlock lastLocatedBlock = null;
|
private LocatedBlock lastLocatedBlock = null;
|
||||||
private boolean isLastBlockComplete = false;
|
private boolean isLastBlockComplete = false;
|
||||||
|
private FileEncryptionInfo fileEncryptionInfo = null;
|
||||||
|
|
||||||
public LocatedBlocks() {
|
public LocatedBlocks() {
|
||||||
fileLength = 0;
|
fileLength = 0;
|
||||||
blocks = null;
|
blocks = null;
|
||||||
underConstruction = false;
|
underConstruction = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** public Constructor */
|
|
||||||
public LocatedBlocks(long flength, boolean isUnderConstuction,
|
public LocatedBlocks(long flength, boolean isUnderConstuction,
|
||||||
List<LocatedBlock> blks,
|
List<LocatedBlock> blks, LocatedBlock lastBlock,
|
||||||
LocatedBlock lastBlock, boolean isLastBlockCompleted) {
|
boolean isLastBlockCompleted, FileEncryptionInfo feInfo) {
|
||||||
fileLength = flength;
|
fileLength = flength;
|
||||||
blocks = blks;
|
blocks = blks;
|
||||||
underConstruction = isUnderConstuction;
|
underConstruction = isUnderConstuction;
|
||||||
this.lastLocatedBlock = lastBlock;
|
this.lastLocatedBlock = lastBlock;
|
||||||
this.isLastBlockComplete = isLastBlockCompleted;
|
this.isLastBlockComplete = isLastBlockCompleted;
|
||||||
|
this.fileEncryptionInfo = feInfo;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -92,13 +94,20 @@ public class LocatedBlocks {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Return ture if file was under construction when
|
* Return true if file was under construction when this LocatedBlocks was
|
||||||
* this LocatedBlocks was constructed, false otherwise.
|
* constructed, false otherwise.
|
||||||
*/
|
*/
|
||||||
public boolean isUnderConstruction() {
|
public boolean isUnderConstruction() {
|
||||||
return underConstruction;
|
return underConstruction;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return the FileEncryptionInfo for the LocatedBlocks
|
||||||
|
*/
|
||||||
|
public FileEncryptionInfo getFileEncryptionInfo() {
|
||||||
|
return fileEncryptionInfo;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Find block containing specified offset.
|
* Find block containing specified offset.
|
||||||
*
|
*
|
||||||
|
|
|
@ -62,7 +62,7 @@ public class SnapshottableDirectoryStatus {
|
||||||
int snapshotNumber, int snapshotQuota, byte[] parentFullPath) {
|
int snapshotNumber, int snapshotQuota, byte[] parentFullPath) {
|
||||||
this.dirStatus = new HdfsFileStatus(0, true, 0, 0, modification_time,
|
this.dirStatus = new HdfsFileStatus(0, true, 0, 0, modification_time,
|
||||||
access_time, permission, owner, group, null, localName, inodeId,
|
access_time, permission, owner, group, null, localName, inodeId,
|
||||||
childrenNum, BlockStoragePolicy.ID_UNSPECIFIED);
|
childrenNum, null, BlockStoragePolicy.ID_UNSPECIFIED);
|
||||||
this.snapshotNumber = snapshotNumber;
|
this.snapshotNumber = snapshotNumber;
|
||||||
this.snapshotQuota = snapshotQuota;
|
this.snapshotQuota = snapshotQuota;
|
||||||
this.parentFullPath = parentFullPath;
|
this.parentFullPath = parentFullPath;
|
||||||
|
|
|
@ -162,8 +162,10 @@ public final class DataTransferSaslUtil {
|
||||||
Configuration saslPropsResolverConf = new Configuration(conf);
|
Configuration saslPropsResolverConf = new Configuration(conf);
|
||||||
saslPropsResolverConf.set(HADOOP_RPC_PROTECTION, qops);
|
saslPropsResolverConf.set(HADOOP_RPC_PROTECTION, qops);
|
||||||
Class<? extends SaslPropertiesResolver> resolverClass = conf.getClass(
|
Class<? extends SaslPropertiesResolver> resolverClass = conf.getClass(
|
||||||
DFS_DATA_TRANSFER_SASL_PROPS_RESOLVER_CLASS_KEY,
|
HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS,
|
||||||
SaslPropertiesResolver.class, SaslPropertiesResolver.class);
|
SaslPropertiesResolver.class, SaslPropertiesResolver.class);
|
||||||
|
resolverClass = conf.getClass(DFS_DATA_TRANSFER_SASL_PROPS_RESOLVER_CLASS_KEY,
|
||||||
|
resolverClass, SaslPropertiesResolver.class);
|
||||||
saslPropsResolverConf.setClass(HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS,
|
saslPropsResolverConf.setClass(HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS,
|
||||||
resolverClass, SaslPropertiesResolver.class);
|
resolverClass, SaslPropertiesResolver.class);
|
||||||
SaslPropertiesResolver resolver = SaslPropertiesResolver.getInstance(
|
SaslPropertiesResolver resolver = SaslPropertiesResolver.getInstance(
|
||||||
|
|
|
@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
|
||||||
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
|
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
|
||||||
import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks;
|
import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks;
|
||||||
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
|
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.EncryptionZoneWithId;
|
||||||
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
|
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
|
||||||
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
|
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
|
||||||
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
|
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
|
||||||
|
@ -178,6 +179,12 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Update
|
||||||
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdatePipelineResponseProto;
|
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdatePipelineResponseProto;
|
||||||
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CheckAccessRequestProto;
|
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CheckAccessRequestProto;
|
||||||
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CheckAccessResponseProto;
|
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CheckAccessResponseProto;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.CreateEncryptionZoneResponseProto;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.CreateEncryptionZoneRequestProto;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.GetEZForPathResponseProto;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.GetEZForPathRequestProto;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ListEncryptionZonesResponseProto;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ListEncryptionZonesRequestProto;
|
||||||
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeIDProto;
|
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeIDProto;
|
||||||
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfoProto;
|
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfoProto;
|
||||||
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto;
|
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto;
|
||||||
|
@ -380,7 +387,8 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
|
||||||
HdfsFileStatus result = server.create(req.getSrc(),
|
HdfsFileStatus result = server.create(req.getSrc(),
|
||||||
PBHelper.convert(req.getMasked()), req.getClientName(),
|
PBHelper.convert(req.getMasked()), req.getClientName(),
|
||||||
PBHelper.convertCreateFlag(req.getCreateFlag()), req.getCreateParent(),
|
PBHelper.convertCreateFlag(req.getCreateFlag()), req.getCreateParent(),
|
||||||
(short) req.getReplication(), req.getBlockSize());
|
(short) req.getReplication(), req.getBlockSize(),
|
||||||
|
PBHelper.convertCipherSuiteProtos(req.getCipherSuitesList()));
|
||||||
|
|
||||||
if (result != null) {
|
if (result != null) {
|
||||||
return CreateResponseProto.newBuilder().setFs(PBHelper.convert(result))
|
return CreateResponseProto.newBuilder().setFs(PBHelper.convert(result))
|
||||||
|
@ -1304,6 +1312,52 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public CreateEncryptionZoneResponseProto createEncryptionZone(
|
||||||
|
RpcController controller, CreateEncryptionZoneRequestProto req)
|
||||||
|
throws ServiceException {
|
||||||
|
try {
|
||||||
|
server.createEncryptionZone(req.getSrc(), req.getKeyName());
|
||||||
|
return CreateEncryptionZoneResponseProto.newBuilder().build();
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new ServiceException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GetEZForPathResponseProto getEZForPath(
|
||||||
|
RpcController controller, GetEZForPathRequestProto req)
|
||||||
|
throws ServiceException {
|
||||||
|
try {
|
||||||
|
GetEZForPathResponseProto.Builder builder =
|
||||||
|
GetEZForPathResponseProto.newBuilder();
|
||||||
|
final EncryptionZoneWithId ret = server.getEZForPath(req.getSrc());
|
||||||
|
builder.setZone(PBHelper.convert(ret));
|
||||||
|
return builder.build();
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new ServiceException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ListEncryptionZonesResponseProto listEncryptionZones(
|
||||||
|
RpcController controller, ListEncryptionZonesRequestProto req)
|
||||||
|
throws ServiceException {
|
||||||
|
try {
|
||||||
|
BatchedEntries<EncryptionZoneWithId> entries = server
|
||||||
|
.listEncryptionZones(req.getId());
|
||||||
|
ListEncryptionZonesResponseProto.Builder builder =
|
||||||
|
ListEncryptionZonesResponseProto.newBuilder();
|
||||||
|
builder.setHasMore(entries.hasMore());
|
||||||
|
for (int i=0; i<entries.size(); i++) {
|
||||||
|
builder.addZones(PBHelper.convert(entries.get(i)));
|
||||||
|
}
|
||||||
|
return builder.build();
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new ServiceException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public SetXAttrResponseProto setXAttr(RpcController controller,
|
public SetXAttrResponseProto setXAttr(RpcController controller,
|
||||||
SetXAttrRequestProto req) throws ServiceException {
|
SetXAttrRequestProto req) throws ServiceException {
|
||||||
|
|
|
@ -24,8 +24,10 @@ import java.util.Arrays;
|
||||||
import java.util.EnumSet;
|
import java.util.EnumSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
|
import com.google.common.collect.Lists;
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
|
import org.apache.hadoop.crypto.CipherSuite;
|
||||||
import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries;
|
import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries;
|
||||||
import org.apache.hadoop.fs.CacheFlag;
|
import org.apache.hadoop.fs.CacheFlag;
|
||||||
import org.apache.hadoop.fs.ContentSummary;
|
import org.apache.hadoop.fs.ContentSummary;
|
||||||
|
@ -52,6 +54,7 @@ import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
|
||||||
import org.apache.hadoop.hdfs.protocol.DatanodeID;
|
import org.apache.hadoop.hdfs.protocol.DatanodeID;
|
||||||
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
|
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
|
||||||
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
|
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.EncryptionZoneWithId;
|
||||||
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
|
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
|
||||||
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
|
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
|
||||||
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
|
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
|
||||||
|
@ -149,6 +152,10 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Update
|
||||||
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdatePipelineRequestProto;
|
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdatePipelineRequestProto;
|
||||||
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CheckAccessRequestProto;
|
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CheckAccessRequestProto;
|
||||||
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetStoragePolicyRequestProto;
|
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetStoragePolicyRequestProto;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.CreateEncryptionZoneRequestProto;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.GetEZForPathRequestProto;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ListEncryptionZonesRequestProto;
|
||||||
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsRequestProto;
|
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsRequestProto;
|
||||||
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.ListXAttrsRequestProto;
|
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.ListXAttrsRequestProto;
|
||||||
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.RemoveXAttrRequestProto;
|
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.RemoveXAttrRequestProto;
|
||||||
|
@ -175,6 +182,11 @@ import org.apache.hadoop.security.token.Token;
|
||||||
import com.google.protobuf.ByteString;
|
import com.google.protobuf.ByteString;
|
||||||
import com.google.protobuf.ServiceException;
|
import com.google.protobuf.ServiceException;
|
||||||
|
|
||||||
|
|
||||||
|
import static org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries;
|
||||||
|
import static org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos
|
||||||
|
.EncryptionZoneWithIdProto;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This class forwards NN's ClientProtocol calls as RPC calls to the NN server
|
* This class forwards NN's ClientProtocol calls as RPC calls to the NN server
|
||||||
* while translating from the parameter types used in ClientProtocol to the
|
* while translating from the parameter types used in ClientProtocol to the
|
||||||
|
@ -252,21 +264,25 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
@Override
|
@Override
|
||||||
public HdfsFileStatus create(String src, FsPermission masked,
|
public HdfsFileStatus create(String src, FsPermission masked,
|
||||||
String clientName, EnumSetWritable<CreateFlag> flag,
|
String clientName, EnumSetWritable<CreateFlag> flag,
|
||||||
boolean createParent, short replication, long blockSize)
|
boolean createParent, short replication, long blockSize,
|
||||||
|
List<CipherSuite> cipherSuites)
|
||||||
throws AccessControlException, AlreadyBeingCreatedException,
|
throws AccessControlException, AlreadyBeingCreatedException,
|
||||||
DSQuotaExceededException, FileAlreadyExistsException,
|
DSQuotaExceededException, FileAlreadyExistsException,
|
||||||
FileNotFoundException, NSQuotaExceededException,
|
FileNotFoundException, NSQuotaExceededException,
|
||||||
ParentNotDirectoryException, SafeModeException, UnresolvedLinkException,
|
ParentNotDirectoryException, SafeModeException, UnresolvedLinkException,
|
||||||
IOException {
|
IOException {
|
||||||
CreateRequestProto req = CreateRequestProto.newBuilder()
|
CreateRequestProto.Builder builder = CreateRequestProto.newBuilder()
|
||||||
.setSrc(src)
|
.setSrc(src)
|
||||||
.setMasked(PBHelper.convert(masked))
|
.setMasked(PBHelper.convert(masked))
|
||||||
.setClientName(clientName)
|
.setClientName(clientName)
|
||||||
.setCreateFlag(PBHelper.convertCreateFlag(flag))
|
.setCreateFlag(PBHelper.convertCreateFlag(flag))
|
||||||
.setCreateParent(createParent)
|
.setCreateParent(createParent)
|
||||||
.setReplication(replication)
|
.setReplication(replication)
|
||||||
.setBlockSize(blockSize)
|
.setBlockSize(blockSize);
|
||||||
.build();
|
if (cipherSuites != null) {
|
||||||
|
builder.addAllCipherSuites(PBHelper.convertCipherSuites(cipherSuites));
|
||||||
|
}
|
||||||
|
CreateRequestProto req = builder.build();
|
||||||
try {
|
try {
|
||||||
CreateResponseProto res = rpcProxy.create(null, req);
|
CreateResponseProto res = rpcProxy.create(null, req);
|
||||||
return res.hasFs() ? PBHelper.convert(res.getFs()) : null;
|
return res.hasFs() ? PBHelper.convert(res.getFs()) : null;
|
||||||
|
@ -1295,7 +1311,62 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
throw ProtobufHelper.getRemoteException(e);
|
throw ProtobufHelper.getRemoteException(e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void createEncryptionZone(String src, String keyName)
|
||||||
|
throws IOException {
|
||||||
|
final CreateEncryptionZoneRequestProto.Builder builder =
|
||||||
|
CreateEncryptionZoneRequestProto.newBuilder();
|
||||||
|
builder.setSrc(src);
|
||||||
|
if (keyName != null && !keyName.isEmpty()) {
|
||||||
|
builder.setKeyName(keyName);
|
||||||
|
}
|
||||||
|
CreateEncryptionZoneRequestProto req = builder.build();
|
||||||
|
try {
|
||||||
|
rpcProxy.createEncryptionZone(null, req);
|
||||||
|
} catch (ServiceException e) {
|
||||||
|
throw ProtobufHelper.getRemoteException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public EncryptionZoneWithId getEZForPath(String src)
|
||||||
|
throws IOException {
|
||||||
|
final GetEZForPathRequestProto.Builder builder =
|
||||||
|
GetEZForPathRequestProto.newBuilder();
|
||||||
|
builder.setSrc(src);
|
||||||
|
final GetEZForPathRequestProto req = builder.build();
|
||||||
|
try {
|
||||||
|
final EncryptionZonesProtos.GetEZForPathResponseProto response =
|
||||||
|
rpcProxy.getEZForPath(null, req);
|
||||||
|
return PBHelper.convert(response.getZone());
|
||||||
|
} catch (ServiceException e) {
|
||||||
|
throw ProtobufHelper.getRemoteException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BatchedEntries<EncryptionZoneWithId> listEncryptionZones(long id)
|
||||||
|
throws IOException {
|
||||||
|
final ListEncryptionZonesRequestProto req =
|
||||||
|
ListEncryptionZonesRequestProto.newBuilder()
|
||||||
|
.setId(id)
|
||||||
|
.build();
|
||||||
|
try {
|
||||||
|
EncryptionZonesProtos.ListEncryptionZonesResponseProto response =
|
||||||
|
rpcProxy.listEncryptionZones(null, req);
|
||||||
|
List<EncryptionZoneWithId> elements =
|
||||||
|
Lists.newArrayListWithCapacity(response.getZonesCount());
|
||||||
|
for (EncryptionZoneWithIdProto p : response.getZonesList()) {
|
||||||
|
elements.add(PBHelper.convert(p));
|
||||||
|
}
|
||||||
|
return new BatchedListEntries<EncryptionZoneWithId>(elements,
|
||||||
|
response.getHasMore());
|
||||||
|
} catch (ServiceException e) {
|
||||||
|
throw ProtobufHelper.getRemoteException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag)
|
public void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
|
|
@ -18,6 +18,8 @@
|
||||||
package org.apache.hadoop.hdfs.protocolPB;
|
package org.apache.hadoop.hdfs.protocolPB;
|
||||||
|
|
||||||
import static com.google.common.base.Preconditions.checkNotNull;
|
import static com.google.common.base.Preconditions.checkNotNull;
|
||||||
|
import static org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos
|
||||||
|
.EncryptionZoneWithIdProto;
|
||||||
|
|
||||||
import java.io.EOFException;
|
import java.io.EOFException;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
@ -52,6 +54,7 @@ import org.apache.hadoop.hdfs.protocol.CacheDirectiveStats;
|
||||||
import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
|
import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
|
||||||
import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
|
import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
|
||||||
import org.apache.hadoop.hdfs.protocol.CachePoolStats;
|
import org.apache.hadoop.hdfs.protocol.CachePoolStats;
|
||||||
|
import org.apache.hadoop.crypto.CipherSuite;
|
||||||
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
|
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
|
||||||
import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks;
|
import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks;
|
||||||
import org.apache.hadoop.hdfs.protocol.DatanodeID;
|
import org.apache.hadoop.hdfs.protocol.DatanodeID;
|
||||||
|
@ -59,7 +62,9 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
|
||||||
import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
|
import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
|
||||||
import org.apache.hadoop.hdfs.protocol.DatanodeLocalInfo;
|
import org.apache.hadoop.hdfs.protocol.DatanodeLocalInfo;
|
||||||
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
|
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.EncryptionZoneWithId;
|
||||||
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
|
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
|
||||||
|
import org.apache.hadoop.fs.FileEncryptionInfo;
|
||||||
import org.apache.hadoop.hdfs.protocol.FsAclPermission;
|
import org.apache.hadoop.hdfs.protocol.FsAclPermission;
|
||||||
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
|
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
|
||||||
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
|
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
|
||||||
|
@ -1179,7 +1184,9 @@ public class PBHelper {
|
||||||
lb.getFileLength(), lb.getUnderConstruction(),
|
lb.getFileLength(), lb.getUnderConstruction(),
|
||||||
PBHelper.convertLocatedBlock(lb.getBlocksList()),
|
PBHelper.convertLocatedBlock(lb.getBlocksList()),
|
||||||
lb.hasLastBlock() ? PBHelper.convert(lb.getLastBlock()) : null,
|
lb.hasLastBlock() ? PBHelper.convert(lb.getLastBlock()) : null,
|
||||||
lb.getIsLastBlockComplete());
|
lb.getIsLastBlockComplete(),
|
||||||
|
lb.hasFileEncryptionInfo() ? convert(lb.getFileEncryptionInfo()) :
|
||||||
|
null);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static LocatedBlocksProto convert(LocatedBlocks lb) {
|
public static LocatedBlocksProto convert(LocatedBlocks lb) {
|
||||||
|
@ -1191,6 +1198,9 @@ public class PBHelper {
|
||||||
if (lb.getLastLocatedBlock() != null) {
|
if (lb.getLastLocatedBlock() != null) {
|
||||||
builder.setLastBlock(PBHelper.convert(lb.getLastLocatedBlock()));
|
builder.setLastBlock(PBHelper.convert(lb.getLastLocatedBlock()));
|
||||||
}
|
}
|
||||||
|
if (lb.getFileEncryptionInfo() != null) {
|
||||||
|
builder.setFileEncryptionInfo(convert(lb.getFileEncryptionInfo()));
|
||||||
|
}
|
||||||
return builder.setFileLength(lb.getFileLength())
|
return builder.setFileLength(lb.getFileLength())
|
||||||
.setUnderConstruction(lb.isUnderConstruction())
|
.setUnderConstruction(lb.isUnderConstruction())
|
||||||
.addAllBlocks(PBHelper.convertLocatedBlock2(lb.getLocatedBlocks()))
|
.addAllBlocks(PBHelper.convertLocatedBlock2(lb.getLocatedBlocks()))
|
||||||
|
@ -1317,6 +1327,7 @@ public class PBHelper {
|
||||||
fs.hasFileId()? fs.getFileId(): INodeId.GRANDFATHER_INODE_ID,
|
fs.hasFileId()? fs.getFileId(): INodeId.GRANDFATHER_INODE_ID,
|
||||||
fs.hasLocations() ? PBHelper.convert(fs.getLocations()) : null,
|
fs.hasLocations() ? PBHelper.convert(fs.getLocations()) : null,
|
||||||
fs.hasChildrenNum() ? fs.getChildrenNum() : -1,
|
fs.hasChildrenNum() ? fs.getChildrenNum() : -1,
|
||||||
|
fs.hasFileEncryptionInfo() ? convert(fs.getFileEncryptionInfo()) : null,
|
||||||
fs.hasStoragePolicy() ? (byte) fs.getStoragePolicy()
|
fs.hasStoragePolicy() ? (byte) fs.getStoragePolicy()
|
||||||
: BlockStoragePolicy.ID_UNSPECIFIED);
|
: BlockStoragePolicy.ID_UNSPECIFIED);
|
||||||
}
|
}
|
||||||
|
@ -1369,6 +1380,9 @@ public class PBHelper {
|
||||||
if (fs.isSymlink()) {
|
if (fs.isSymlink()) {
|
||||||
builder.setSymlink(ByteString.copyFrom(fs.getSymlinkInBytes()));
|
builder.setSymlink(ByteString.copyFrom(fs.getSymlinkInBytes()));
|
||||||
}
|
}
|
||||||
|
if (fs.getFileEncryptionInfo() != null) {
|
||||||
|
builder.setFileEncryptionInfo(convert(fs.getFileEncryptionInfo()));
|
||||||
|
}
|
||||||
if (fs instanceof HdfsLocatedFileStatus) {
|
if (fs instanceof HdfsLocatedFileStatus) {
|
||||||
final HdfsLocatedFileStatus lfs = (HdfsLocatedFileStatus) fs;
|
final HdfsLocatedFileStatus lfs = (HdfsLocatedFileStatus) fs;
|
||||||
LocatedBlocks locations = lfs.getBlockLocations();
|
LocatedBlocks locations = lfs.getBlockLocations();
|
||||||
|
@ -2262,7 +2276,7 @@ public class PBHelper {
|
||||||
}
|
}
|
||||||
return xAttrs;
|
return xAttrs;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static List<XAttr> convert(GetXAttrsResponseProto a) {
|
public static List<XAttr> convert(GetXAttrsResponseProto a) {
|
||||||
List<XAttrProto> xAttrs = a.getXAttrsList();
|
List<XAttrProto> xAttrs = a.getXAttrsList();
|
||||||
return convertXAttrs(xAttrs);
|
return convertXAttrs(xAttrs);
|
||||||
|
@ -2293,6 +2307,18 @@ public class PBHelper {
|
||||||
return builder.build();
|
return builder.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static EncryptionZoneWithIdProto convert(EncryptionZoneWithId zone) {
|
||||||
|
return EncryptionZoneWithIdProto.newBuilder()
|
||||||
|
.setId(zone.getId())
|
||||||
|
.setKeyName(zone.getKeyName())
|
||||||
|
.setPath(zone.getPath()).build();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static EncryptionZoneWithId convert(EncryptionZoneWithIdProto proto) {
|
||||||
|
return new EncryptionZoneWithId(proto.getPath(), proto.getKeyName(),
|
||||||
|
proto.getId());
|
||||||
|
}
|
||||||
|
|
||||||
public static ShortCircuitShmSlotProto convert(SlotId slotId) {
|
public static ShortCircuitShmSlotProto convert(SlotId slotId) {
|
||||||
return ShortCircuitShmSlotProto.newBuilder().
|
return ShortCircuitShmSlotProto.newBuilder().
|
||||||
setShmId(convert(slotId.getShmId())).
|
setShmId(convert(slotId.getShmId())).
|
||||||
|
@ -2316,5 +2342,75 @@ public class PBHelper {
|
||||||
public static ShmId convert(ShortCircuitShmIdProto shmId) {
|
public static ShmId convert(ShortCircuitShmIdProto shmId) {
|
||||||
return new ShmId(shmId.getHi(), shmId.getLo());
|
return new ShmId(shmId.getHi(), shmId.getLo());
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
|
public static HdfsProtos.CipherSuite convert(CipherSuite suite) {
|
||||||
|
switch (suite) {
|
||||||
|
case UNKNOWN:
|
||||||
|
return HdfsProtos.CipherSuite.UNKNOWN;
|
||||||
|
case AES_CTR_NOPADDING:
|
||||||
|
return HdfsProtos.CipherSuite.AES_CTR_NOPADDING;
|
||||||
|
default:
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static CipherSuite convert(HdfsProtos.CipherSuite proto) {
|
||||||
|
switch (proto) {
|
||||||
|
case AES_CTR_NOPADDING:
|
||||||
|
return CipherSuite.AES_CTR_NOPADDING;
|
||||||
|
default:
|
||||||
|
// Set to UNKNOWN and stash the unknown enum value
|
||||||
|
CipherSuite suite = CipherSuite.UNKNOWN;
|
||||||
|
suite.setUnknownValue(proto.getNumber());
|
||||||
|
return suite;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static List<HdfsProtos.CipherSuite> convertCipherSuites
|
||||||
|
(List<CipherSuite> suites) {
|
||||||
|
if (suites == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
List<HdfsProtos.CipherSuite> protos =
|
||||||
|
Lists.newArrayListWithCapacity(suites.size());
|
||||||
|
for (CipherSuite suite : suites) {
|
||||||
|
protos.add(convert(suite));
|
||||||
|
}
|
||||||
|
return protos;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static List<CipherSuite> convertCipherSuiteProtos(
|
||||||
|
List<HdfsProtos.CipherSuite> protos) {
|
||||||
|
List<CipherSuite> suites = Lists.newArrayListWithCapacity(protos.size());
|
||||||
|
for (HdfsProtos.CipherSuite proto : protos) {
|
||||||
|
suites.add(convert(proto));
|
||||||
|
}
|
||||||
|
return suites;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static HdfsProtos.FileEncryptionInfoProto convert(
|
||||||
|
FileEncryptionInfo info) {
|
||||||
|
if (info == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return HdfsProtos.FileEncryptionInfoProto.newBuilder()
|
||||||
|
.setSuite(convert(info.getCipherSuite()))
|
||||||
|
.setKey(getByteString(info.getEncryptedDataEncryptionKey()))
|
||||||
|
.setIv(getByteString(info.getIV()))
|
||||||
|
.setEzKeyVersionName(info.getEzKeyVersionName())
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static FileEncryptionInfo convert(
|
||||||
|
HdfsProtos.FileEncryptionInfoProto proto) {
|
||||||
|
if (proto == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
CipherSuite suite = convert(proto.getSuite());
|
||||||
|
byte[] key = proto.getKey().toByteArray();
|
||||||
|
byte[] iv = proto.getIv().toByteArray();
|
||||||
|
String ezKeyVersionName = proto.getEzKeyVersionName();
|
||||||
|
return new FileEncryptionInfo(suite, key, iv, ezKeyVersionName);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
|
@ -53,6 +53,8 @@ import org.apache.hadoop.hdfs.protocol.BlockListAsLongs.BlockReportIterator;
|
||||||
import org.apache.hadoop.hdfs.protocol.DatanodeID;
|
import org.apache.hadoop.hdfs.protocol.DatanodeID;
|
||||||
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
|
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
|
||||||
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
|
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
|
||||||
|
import org.apache.hadoop.fs.FileEncryptionInfo;
|
||||||
|
|
||||||
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
|
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
|
||||||
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
|
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
|
||||||
import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
|
import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
|
||||||
|
@ -846,14 +848,15 @@ public class BlockManager {
|
||||||
public LocatedBlocks createLocatedBlocks(final BlockInfo[] blocks,
|
public LocatedBlocks createLocatedBlocks(final BlockInfo[] blocks,
|
||||||
final long fileSizeExcludeBlocksUnderConstruction,
|
final long fileSizeExcludeBlocksUnderConstruction,
|
||||||
final boolean isFileUnderConstruction, final long offset,
|
final boolean isFileUnderConstruction, final long offset,
|
||||||
final long length, final boolean needBlockToken, final boolean inSnapshot)
|
final long length, final boolean needBlockToken,
|
||||||
|
final boolean inSnapshot, FileEncryptionInfo feInfo)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
assert namesystem.hasReadLock();
|
assert namesystem.hasReadLock();
|
||||||
if (blocks == null) {
|
if (blocks == null) {
|
||||||
return null;
|
return null;
|
||||||
} else if (blocks.length == 0) {
|
} else if (blocks.length == 0) {
|
||||||
return new LocatedBlocks(0, isFileUnderConstruction,
|
return new LocatedBlocks(0, isFileUnderConstruction,
|
||||||
Collections.<LocatedBlock>emptyList(), null, false);
|
Collections.<LocatedBlock>emptyList(), null, false, feInfo);
|
||||||
} else {
|
} else {
|
||||||
if (LOG.isDebugEnabled()) {
|
if (LOG.isDebugEnabled()) {
|
||||||
LOG.debug("blocks = " + java.util.Arrays.asList(blocks));
|
LOG.debug("blocks = " + java.util.Arrays.asList(blocks));
|
||||||
|
@ -878,7 +881,7 @@ public class BlockManager {
|
||||||
}
|
}
|
||||||
return new LocatedBlocks(
|
return new LocatedBlocks(
|
||||||
fileSizeExcludeBlocksUnderConstruction, isFileUnderConstruction,
|
fileSizeExcludeBlocksUnderConstruction, isFileUnderConstruction,
|
||||||
locatedblocks, lastlb, isComplete);
|
locatedblocks, lastlb, isComplete, feInfo);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -294,5 +294,10 @@ public final class HdfsServerConstants {
|
||||||
|
|
||||||
public static final String NAMENODE_LEASE_HOLDER = "HDFS_NameNode";
|
public static final String NAMENODE_LEASE_HOLDER = "HDFS_NameNode";
|
||||||
public static final long NAMENODE_LEASE_RECHECK_INTERVAL = 2000;
|
public static final long NAMENODE_LEASE_RECHECK_INTERVAL = 2000;
|
||||||
|
|
||||||
|
public static final String CRYPTO_XATTR_ENCRYPTION_ZONE =
|
||||||
|
"raw.hdfs.crypto.encryption.zone";
|
||||||
|
public static final String CRYPTO_XATTR_FILE_ENCRYPTION_INFO =
|
||||||
|
"raw.hdfs.crypto.file.encryption.info";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -576,7 +576,9 @@ class DataXceiver extends Receiver implements Runnable {
|
||||||
// forward the original version of the block to downstream mirrors, so
|
// forward the original version of the block to downstream mirrors, so
|
||||||
// make a copy here.
|
// make a copy here.
|
||||||
final ExtendedBlock originalBlock = new ExtendedBlock(block);
|
final ExtendedBlock originalBlock = new ExtendedBlock(block);
|
||||||
block.setNumBytes(dataXceiverServer.estimateBlockSize);
|
if (block.getNumBytes() == 0) {
|
||||||
|
block.setNumBytes(dataXceiverServer.estimateBlockSize);
|
||||||
|
}
|
||||||
LOG.info("Receiving " + block + " src: " + remoteAddress + " dest: "
|
LOG.info("Receiving " + block + " src: " + remoteAddress + " dest: "
|
||||||
+ localAddress);
|
+ localAddress);
|
||||||
|
|
||||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue