From 754718aed56480135acd0f32878f1fe4607c34df Mon Sep 17 00:00:00 2001 From: cnauroth Date: Thu, 9 Oct 2014 11:55:59 -0700 Subject: [PATCH] HDFS-7195. Update user doc of secure mode about Datanodes don't require root or jsvc. Contributed by Chris Nauroth. (cherry picked from commit 4e544c84e9e2cf3fb031cd8797bac4e8bcc7eca9) --- .../hadoop-common/src/main/conf/hadoop-env.sh | 11 ++++- .../src/site/apt/SecureMode.apt.vm | 47 ++++++++++++++++++- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh index e964ec08217..f60b65a0b69 100644 --- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh +++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh @@ -24,7 +24,10 @@ # The java implementation to use. export JAVA_HOME=${JAVA_HOME} -# The jsvc implementation to use. Jsvc is required to run secure datanodes. +# The jsvc implementation to use. Jsvc is required to run secure datanodes +# that bind to privileged ports to provide authentication of data transfer +# protocol. Jsvc is not required if SASL is configured for authentication of +# data transfer protocol using non-privileged ports. #export JSVC_HOME=${JSVC_HOME} export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/etc/hadoop"} @@ -58,7 +61,11 @@ export HADOOP_PORTMAP_OPTS="-Xmx512m $HADOOP_PORTMAP_OPTS" export HADOOP_CLIENT_OPTS="-Xmx512m $HADOOP_CLIENT_OPTS" #HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData $HADOOP_JAVA_PLATFORM_OPTS" -# On secure datanodes, user to run the datanode as after dropping privileges +# On secure datanodes, user to run the datanode as after dropping privileges. +# This **MUST** be uncommented to enable secure HDFS if using privileged ports +# to provide authentication of data transfer protocol. This **MUST NOT** be +# defined if SASL is configured for authentication of data transfer protocol +# using non-privileged ports. export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER} # Where log files are stored. $HADOOP_HOME/logs by default. diff --git a/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm index 54473e6d3d8..2085d5b8d55 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm @@ -244,7 +244,7 @@ KVNO Timestamp Principal 10.222.0.0-15 and 10.113.221.221 can impersonate any user belonging to any group. - ---- +---- hadoop.proxyuser.oozie.hosts 10.222.0.0/16,10.113.221.221 @@ -273,6 +273,33 @@ KVNO Timestamp Principal You must specify <<>> and <<>> as environment variables on start up (in hadoop-env.sh). + As of version 2.6.0, SASL can be used to authenticate the data transfer + protocol. In this configuration, it is no longer required for secured clusters + to start the DataNode as root using jsvc and bind to privileged ports. To + enable SASL on data transfer protocol, set <<>> + in hdfs-site.xml, set a non-privileged port for <<>>, set + <<>> to and make sure the + <<>> environment variable is not defined. Note that it + is not possible to use SASL on data transfer protocol if + <<>> is set to a privileged port. This is required for + backwards-compatibility reasons. + + In order to migrate an existing cluster that used root authentication to start + using SASL instead, first ensure that version 2.6.0 or later has been deployed + to all cluster nodes as well as any external applications that need to connect + to the cluster. Only versions 2.6.0 and later of the HDFS client can connect + to a DataNode that uses SASL for authentication of data transfer protocol, so + it is vital that all callers have the correct version before migrating. After + version 2.6.0 or later has been deployed everywhere, update configuration of + any external applications to enable SASL. If an HDFS client is enabled for + SASL, then it can connect successfully to a DataNode running with either root + authentication or SASL authentication. Changing configuration for all clients + guarantees that subsequent configuration changes on DataNodes will not disrupt + the applications. Finally, each individual DataNode can be migrated by + changing its configuration and restarting. It is acceptable to have a mix of + some DataNodes running with root authentication and some DataNodes running with + SASL authentication temporarily during this migration period, because an HDFS + client enabled for SASL can connect to both. * Data confidentiality @@ -391,6 +418,10 @@ Configuration for <<>> | <<>> | or or | | | | | HTTPS_ONLY turns off http access. This option takes precedence over | | | | the deprecated configuration dfs.https.enable and hadoop.ssl.enabled. | +| | | If using SASL to authenticate data transfer protocol instead of | +| | | running DataNode as root and using privileged ports, then this property | +| | | must be set to to guarantee authentication of HTTP servers. | +| | | (See <<>>.) | *-------------------------+-------------------------+------------------------+ | <<>> | | | *-------------------------+-------------------------+------------------------+ @@ -440,6 +471,9 @@ Configuration for <<>> | | | Secure DataNode must use privileged port | | | | in order to assure that the server was started securely. | | | | This means that the server must be started via jsvc. | +| | | Alternatively, this must be set to a non-privileged port if using SASL | +| | | to authenticate data transfer protocol. | +| | | (See <<>>.) | *-------------------------+-------------------------+------------------------+ | <<>> | <0.0.0.0:1006> | | | | | Secure DataNode must use privileged port | @@ -457,6 +491,17 @@ Configuration for <<>> | <<>> | | | | | | set to <<>> when using data encryption | *-------------------------+-------------------------+------------------------+ +| <<>> | | | +| | | : authentication only \ +| | | : integrity check in addition to authentication \ +| | | : data encryption in addition to integrity | +| | | This property is unspecified by default. Setting this property enables | +| | | SASL for authentication of data transfer protocol. If this is enabled, | +| | | then <<>> must use a non-privileged port, | +| | | <<>> must be set to and the | +| | | <<>> environment variable must be undefined when | +| | | starting the DataNode process. | +*-------------------------+-------------------------+------------------------+ Configuration for <<>> diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 85a2ad9baff..9f02ea4a2a6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -280,6 +280,9 @@ Release 2.6.0 - UNRELEASED HDFS-7217. Better batching of IBRs. (kihwal) + HDFS-7195. Update user doc of secure mode about Datanodes don't require root + or jsvc. (cnauroth) + OPTIMIZATIONS HDFS-6690. Deduplicate xattr names in memory. (wang)