Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-11418. HttpFS should support old SSL clients. Contributed by John Zhuge. 

Change-Id: I05b1228f7f168e9b5904365ba9e81be3f29dce74
This commit is contained in:
John Zhuge 2017-02-15 11:08:57 -08:00 committed by John Zhuge
parent 133857ba04
commit cacaa299cf
7 changed files with 168 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
View File

@ -526,6 +526,9 @@
<delete dir="${project.build.directory}/tomcat.exp"/>
<delete dir="${httpfs.tomcat.dist.dir}/webapps"/>
<mkdir dir="${httpfs.tomcat.dist.dir}/webapps"/>
<delete file="${httpfs.tomcat.dist.dir}/conf/catalina-default.properties"/>
<copy file="${basedir}/src/main/tomcat/catalina-default.properties"
toDir="${httpfs.tomcat.dist.dir}/conf"/>
<delete file="${httpfs.tomcat.dist.dir}/conf/server.xml"/>
<copy file="${basedir}/src/main/tomcat/server.xml"
toDir="${httpfs.tomcat.dist.dir}/conf"/>

4
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh
View File

@ -44,6 +44,10 @@
#
# export HTTPFS_SSL_ENABLED=false
# The comma separated list of encryption ciphers for SSL
#
# export HTTPFS_SSL_CIPHERS=
# The maximum size of Tomcat HTTP header
#
# export HTTPFS_MAX_HTTP_HEADER_SIZE=65536

35
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh
View File

@ -56,7 +56,10 @@ print "Setting HTTPFS_HOME: ${HTTPFS_HOME}"
if [ -e "${HTTPFS_HOME}/bin/httpfs-env.sh" ]; then
print "Sourcing: ${HTTPFS_HOME}/bin/httpfs-env.sh"
source ${HTTPFS_HOME}/bin/httpfs-env.sh
grep "^ *export " ${HTTPFS_HOME}/bin/httpfs-env.sh | sed 's/ *export/ setting/'
if [ "${HTTPFS_SILENT}" != "true" ]; then
grep "^ *export " "${HTTPFS_HOME}/bin/httpfs-env.sh" |
sed 's/ *export/ setting/'
fi
fi
# verify that the sourced env file didn't change HTTPFS_HOME
@ -81,7 +84,10 @@ httpfs_config=${HTTPFS_CONFIG}
if [ -e "${HTTPFS_CONFIG}/httpfs-env.sh" ]; then
print "Sourcing: ${HTTPFS_CONFIG}/httpfs-env.sh"
source ${HTTPFS_CONFIG}/httpfs-env.sh
grep "^ *export " ${HTTPFS_CONFIG}/httpfs-env.sh | sed 's/ *export/ setting/'
if [ "${HTTPFS_SILENT}" != "true" ]; then
grep "^ *export " "${HTTPFS_CONFIG}/httpfs-env.sh" |
sed 's/ *export/ setting/'
fi
fi
# verify that the sourced env file didn't change HTTPFS_HOME
@ -150,6 +156,31 @@ else
print "Using HTTPFS_SSL_ENABLED: ${HTTPFS_SSL_ENABLED}"
fi
if [ "${HTTPFS_SSL_CIPHERS}" = "" ]; then
export HTTPFS_SSL_CIPHERS="TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384"
HTTPFS_SSL_CIPHERS+=",TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"
HTTPFS_SSL_CIPHERS+=",TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384"
HTTPFS_SSL_CIPHERS+=",TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256"
HTTPFS_SSL_CIPHERS+=",TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384"
HTTPFS_SSL_CIPHERS+=",TLS_ECDH_RSA_WITH_AES_256_CBC_SHA"
HTTPFS_SSL_CIPHERS+=",TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256"
HTTPFS_SSL_CIPHERS+=",TLS_ECDH_RSA_WITH_AES_128_CBC_SHA"
HTTPFS_SSL_CIPHERS+=",TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA"
HTTPFS_SSL_CIPHERS+=",TLS_RSA_WITH_AES_256_CBC_SHA256"
HTTPFS_SSL_CIPHERS+=",TLS_RSA_WITH_AES_256_CBC_SHA"
HTTPFS_SSL_CIPHERS+=",TLS_RSA_WITH_AES_128_CBC_SHA256"
HTTPFS_SSL_CIPHERS+=",TLS_RSA_WITH_AES_128_CBC_SHA"
HTTPFS_SSL_CIPHERS+=",TLS_RSA_WITH_3DES_EDE_CBC_SHA"
HTTPFS_SSL_CIPHERS+=",TLS_DHE_RSA_WITH_AES_256_CBC_SHA256"
HTTPFS_SSL_CIPHERS+=",TLS_DHE_RSA_WITH_AES_256_CBC_SHA"
HTTPFS_SSL_CIPHERS+=",TLS_DHE_RSA_WITH_AES_128_CBC_SHA256"
HTTPFS_SSL_CIPHERS+=",TLS_DHE_RSA_WITH_AES_128_CBC_SHA"
HTTPFS_SSL_CIPHERS+=",TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA"
print "Setting HTTPFS_SSL_CIPHERS: ${HTTPFS_SSL_CIPHERS}"
else
print "Using HTTPFS_SSL_CIPHERS: ${HTTPFS_SSL_CIPHERS}"
fi
if [ "${HTTPFS_SSL_KEYSTORE_FILE}" = "" ]; then
export HTTPFS_SSL_KEYSTORE_FILE=${HOME}/.keystore
print "Setting HTTPFS_SSL_KEYSTORE_FILE: ${HTTPFS_SSL_KEYSTORE_FILE}"

47
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
View File

@ -29,6 +29,8 @@ done
BASEDIR=`dirname ${PRG}`
BASEDIR=`cd ${BASEDIR}/..;pwd`
HTTPFS_SILENT=${HTTPFS_SILENT:-true}
source ${HADOOP_LIBEXEC_DIR:-${BASEDIR}/libexec}/httpfs-config.sh
# The Java System property 'httpfs.http.port' it is not used by HttpFS,
@ -37,21 +39,42 @@ source ${HADOOP_LIBEXEC_DIR:-${BASEDIR}/libexec}/httpfs-config.sh
print "Using CATALINA_OPTS: ${CATALINA_OPTS}"
catalina_opts="-Dproc_httpfs";
catalina_opts="${catalina_opts} -Dhttpfs.home.dir=${HTTPFS_HOME}";
catalina_opts="${catalina_opts} -Dhttpfs.config.dir=${HTTPFS_CONFIG}";
catalina_opts="${catalina_opts} -Dhttpfs.log.dir=${HTTPFS_LOG}";
catalina_opts="${catalina_opts} -Dhttpfs.temp.dir=${HTTPFS_TEMP}";
catalina_opts="${catalina_opts} -Dhttpfs.admin.port=${HTTPFS_ADMIN_PORT}";
catalina_opts="${catalina_opts} -Dhttpfs.http.port=${HTTPFS_HTTP_PORT}";
catalina_opts="${catalina_opts} -Dhttpfs.http.hostname=${HTTPFS_HTTP_HOSTNAME}";
catalina_opts="${catalina_opts} -Dhttpfs.ssl.enabled=${HTTPFS_SSL_ENABLED}";
catalina_opts="${catalina_opts} -Dhttpfs.ssl.keystore.file=${HTTPFS_SSL_KEYSTORE_FILE}";
catalina_opts="${catalina_opts} -Dhttpfs.ssl.keystore.pass=${HTTPFS_SSL_KEYSTORE_PASS}";
catalina_opts="${catalina_opts} -Dhttpfs.log.dir=${HTTPFS_LOG}"
print "Adding to CATALINA_OPTS: ${catalina_opts}"
export CATALINA_OPTS="${CATALINA_OPTS} ${catalina_opts}"
catalina_init_properties() {
cp "${CATALINA_BASE}/conf/catalina-default.properties" \
"${CATALINA_BASE}/conf/catalina.properties"
}
catalina_set_property() {
local key=$1
local value=$2
[[ -z "${value}" ]] && return
local disp_value="${3:-${value}}"
print "Setting catalina property ${key} to ${disp_value}"
echo "${key}=${value}" >> "${CATALINA_BASE}/conf/catalina.properties"
}
if [[ "${1}" = "start" || "${1}" = "run" ]]; then
catalina_init_properties
catalina_set_property "httpfs.home.dir" "${HTTPFS_HOME}"
catalina_set_property "httpfs.config.dir" "${HTTPFS_CONFIG}"
catalina_set_property "httpfs.temp.dir" "${HTTPFS_TEMP}"
catalina_set_property "httpfs.admin.port" "${HTTPFS_ADMIN_PORT}"
catalina_set_property "httpfs.http.port" "${HTTPFS_HTTP_PORT}"
catalina_set_property "httpfs.http.hostname" "${HTTPFS_HTTP_HOSTNAME}"
catalina_set_property "httpfs.ssl.enabled" "${HTTPFS_SSL_ENABLED}"
catalina_set_property "httpfs.ssl.ciphers" "${HTTPFS_SSL_CIPHERS}"
catalina_set_property "httpfs.ssl.keystore.file" \
"${HTTPFS_SSL_KEYSTORE_FILE}"
catalina_set_property "httpfs.ssl.keystore.pass" \
"${HTTPFS_SSL_KEYSTORE_PASS}" "<redacted>"
fi
# A bug in catalina.sh script does not use CATALINA_OPTS for stopping the server
#
if [ "${1}" = "stop" ]; then
@ -59,8 +82,8 @@ if [ "${1}" = "stop" ]; then
fi
if [ "${HTTPFS_SILENT}" != "true" ]; then
exec ${HTTPFS_CATALINA_HOME}/bin/catalina.sh "$@"
exec "${HTTPFS_CATALINA_HOME}/bin/catalina.sh" "$@"
else
exec ${HTTPFS_CATALINA_HOME}/bin/catalina.sh "$@" > /dev/null
exec "${HTTPFS_CATALINA_HOME}/bin/catalina.sh" "$@" > /dev/null
fi

87
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/catalina-default.properties Normal file
View File

@ -0,0 +1,87 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# List of comma-separated packages that start with or equal this string
# will cause a security exception to be thrown when
# passed to checkPackageAccess unless the
# corresponding RuntimePermission ("accessClassInPackage."+package) has
# been granted.
package.access=sun.,org.apache.catalina.,org.apache.coyote.,org.apache.jasper.,\
org.apache.naming.resources.,org.apache.tomcat.,sun.beans.
#
# List of comma-separated packages that start with or equal this string
# will cause a security exception to be thrown when
# passed to checkPackageDefinition unless the
# corresponding RuntimePermission ("defineClassInPackage."+package) has
# been granted.
#
# by default, no packages are restricted for definition, and none of
# the class loaders supplied with the JDK call checkPackageDefinition.
#
package.definition=sun.,java.,org.apache.catalina.,org.apache.coyote.,\
org.apache.jasper.,org.apache.naming.,org.apache.tomcat.
#
#
# List of comma-separated paths defining the contents of the "common"
# classloader. Prefixes should be used to define what is the repository type.
# Path may be relative to the CATALINA_HOME or CATALINA_BASE path or absolute.
# If left as blank,the JVM system loader will be used as Catalina's "common"
# loader.
# Examples:
# "foo": Add this folder as a class repository
# "foo/*.jar": Add all the JARs of the specified folder as class
# repositories
# "foo/bar.jar": Add bar.jar as a class repository
common.loader=${catalina.base}/lib,${catalina.base}/lib/*.jar,${catalina.home}/lib,${catalina.home}/lib/*.jar
#
# List of comma-separated paths defining the contents of the "server"
# classloader. Prefixes should be used to define what is the repository type.
# Path may be relative to the CATALINA_HOME or CATALINA_BASE path or absolute.
# If left as blank, the "common" loader will be used as Catalina's "server"
# loader.
# Examples:
# "foo": Add this folder as a class repository
# "foo/*.jar": Add all the JARs of the specified folder as class
# repositories
# "foo/bar.jar": Add bar.jar as a class repository
server.loader=
#
# List of comma-separated paths defining the contents of the "shared"
# classloader. Prefixes should be used to define what is the repository type.
# Path may be relative to the CATALINA_BASE path or absolute. If left as blank,
# the "common" loader will be used as Catalina's "shared" loader.
# Examples:
# "foo": Add this folder as a class repository
# "foo/*.jar": Add all the JARs of the specified folder as class
# repositories
# "foo/bar.jar": Add bar.jar as a class repository
# Please note that for single jars, e.g. bar.jar, you need the URL form
# starting with file:.
shared.loader=
#
# String cache configuration.
tomcat.util.buf.StringCache.byte.enabled=true
#tomcat.util.buf.StringCache.char.enabled=true
#tomcat.util.buf.StringCache.trainThreshold=500000
#tomcat.util.buf.StringCache.cacheSize=5000
#
# Copy catalina-default.properties to catalina.properties
# and then append the custom properties.

1
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/ssl-server.xml
View File

@ -72,6 +72,7 @@
maxThreads="150" scheme="https" secure="true"
maxHttpHeaderSize="${httpfs.max.http.header.size}"
clientAuth="false" sslEnabledProtocols="TLSv1,TLSv1.1,TLSv1.2,SSLv2Hello"
ciphers="${httpfs.ssl.ciphers}"
keystoreFile="${httpfs.ssl.keystore.file}"
keystorePass="${httpfs.ssl.keystore.pass}"/>

6
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm
View File

@ -120,4 +120,8 @@ Start HttpFS. It should work over HTTPS.
Using the Hadoop `FileSystem` API or the Hadoop FS shell, use the `swebhdfs://` scheme. Make sure the JVM is picking up the truststore containing the public key of the SSL certificate if using a self-signed certificate.
NOTE: Some old SSL clients may use weak ciphers that are not supported by the HttpFS server. It is recommended to upgrade the SSL client.
In order to support some old SSL clients, the default encryption ciphers
include a few relatively weaker ciphers. Set environment variable
`HTTPFS_SSL_CIPHERS` or property `httpfs.ssl.ciphers` to override. The value
is a comma separated list of ciphers documented in this
[Tomcat Wiki](https://wiki.apache.org/tomcat/Security/Ciphers).