From 914e93beebc9f2c6eec7282c1fedcaf9f15d84df Mon Sep 17 00:00:00 2001 From: Xiaoyu Yao Date: Fri, 18 May 2018 13:09:17 -0700 Subject: [PATCH] HDDS-7. Enable kerberos auth for Ozone client in hadoop rpc. Contributed by Ajay Kumar. --- .../src/test/compose/compose-secure/.env | 17 ++++ .../compose-secure/docker-compose.yaml | 66 +++++++++++++ .../test/compose/compose-secure/docker-config | 66 +++++++++++++ .../acceptance/ozone-secure.robot | 95 +++++++++++++++++++ .../hadoop/ozone/client/rest/RestClient.java | 4 +- .../hadoop/ozone/client/rpc/RpcClient.java | 7 +- 6 files changed, 249 insertions(+), 6 deletions(-) create mode 100644 hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env create mode 100644 hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml create mode 100644 hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config create mode 100644 hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env new file mode 100644 index 00000000000..32547356cb9 --- /dev/null +++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +OZONEDIR=../../../hadoop-dist/target/ozone \ No newline at end of file diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml new file mode 100644 index 00000000000..2661163c0d6 --- /dev/null +++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3" +services: + ozone.kdc: + image: ahadoop/kdc:v1 + namenode: + image: ahadoop/ozone:v1 + hostname: namenode + volumes: + - ${OZONEDIR}:/opt/hadoop + ports: + - 9000:9000 + environment: + ENSURE_NAMENODE_DIR: /data/namenode + env_file: + - ./docker-config + command: ["/opt/hadoop/bin/hdfs","namenode"] + datanode: + image: ahadoop/ozone:v1 + hostname: datanode + volumes: + - ${OZONEDIR}:/opt/hadoop + ports: + - 9874 + env_file: + - ./docker-config + command: ["/opt/hadoop/bin/ozone","datanode"] + ksm: + image: ahadoop/ozone:v1 + hostname: ksm + volumes: + - ${OZONEDIR}:/opt/hadoop + ports: + - 9874:9874 + environment: + ENSURE_KSM_INITIALIZED: /data/metadata/ksm/current/VERSION + env_file: + - ./docker-config + command: ["/opt/hadoop/bin/ozone","ksm"] + scm: + image: ahadoop/ozone:v1 + hostname: scm + volumes: + - ${OZONEDIR}:/opt/hadoop + ports: + - 9876:9876 + env_file: + - ./docker-config + environment: + ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + command: ["/opt/hadoop/bin/ozone","scm"] diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config new file mode 100644 index 00000000000..678c75af7c9 --- /dev/null +++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +OZONE-SITE.XML_ozone.ksm.address=ksm +OZONE-SITE.XML_ozone.scm.names=scm +OZONE-SITE.XML_ozone.enabled=True +OZONE-SITE.XML_hdds.scm.datanode.id=/data/datanode.id +OZONE-SITE.XML_hdds.scm.block.client.address=scm +OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata +OZONE-SITE.XML_ozone.handler.type=distributed +OZONE-SITE.XML_hdds.scm.client.address=scm +OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService +OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM +OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab +OZONE-SITE.XML_ozone.ksm.kerberos.principal=ksm/ksm@EXAMPLE.COM +OZONE-SITE.XML_ozone.ksm.kerberos.keytab.file=/etc/security/keytabs/ksm.keytab +OZONE-SITE.XML_ozone.security.enabled=true +OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.principal=HTTP/scm@EXAMPLE.COM +OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab +OZONE-SITE.XML_ozone.ksm.web.authentication.kerberos.principal=HTTP/ksm@EXAMPLE.COM +OZONE-SITE.XML_ozone.ksm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab +OZONE-SITE.XML_ozone.scm.block.client.address=scm +OZONE-SITE.XML_ozone.scm.client.address=scm +HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode +HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService +HDFS-SITE.XML_dfs.block.access.token.enable=true +HDFS-SITE.XML_dfs.namenode.kerberos.principal=nn/namenode@EXAMPLE.COM +HDFS-SITE.XML_dfs.namenode.keytab.file=/etc/security/keytabs/nn.keytab +HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/datanode@EXAMPLE.COM +HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab +HDFS-SITE.XML_dfs.namenode.kerberos.internal.spnego.principal=HTTP/namenode@EXAMPLE.COM +HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM +HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab +HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019 +HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012 +HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000 +CORE-SITE.XML_dfs.data.transfer.protection=authentication +CORE-SITE.XML_hadoop.security.authentication=kerberos +CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1@$0](.*)s/.*/root/ +LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout +LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender +LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n + +OZONE_DATANODE_SECURE_USER=root +CONF_DIR=/etc/security/keytabs +KERBEROS_KEYTABS=dn nn ksm scm HTTP testuser +KERBEROS_KEYSTORES=hadoop +KERBEROS_SERVER=ozone.kdc +JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ +JSVC_HOME=/usr/bin +SLEEP_SECONDS=10 +KERBEROS_ENABLED=true diff --git a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot new file mode 100644 index 00000000000..4a789804fe9 --- /dev/null +++ b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Smoke test to start cluster with docker-compose environments. +Library OperatingSystem +Suite Setup Startup Ozone Cluster +Suite Teardown Teardown Ozone Cluster + +*** Variables *** +${COMMON_REST_HEADER} -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE root" +${version} + +*** Test Cases *** + +Daemons are running + Is daemon running ksm + Is daemon running scm + Is daemon running datanode + Is daemon running ozone.kdc + +Check if datanode is connected to the scm + Wait Until Keyword Succeeds 3min 5sec Have healthy datanodes 1 + +Test rest interface + ${result} = Execute on 0 datanode curl -i -X POST ${COMMON_RESTHEADER} "http://localhost:9880/volume1" + Should contain ${result} 201 Created + ${result} = Execute on 0 datanode curl -i -X POST ${COMMON_RESTHEADER} "http://localhost:9880/volume1/bucket1" + Should contain ${result} 201 Created + ${result} = Execute on 0 datanode curl -i -X DELETE ${COMMON_RESTHEADER} "http://localhost:9880/volume1/bucket1" + Should contain ${result} 200 OK + ${result} = Execute on 0 datanode curl -i -X DELETE ${COMMON_RESTHEADER} "http://localhost:9880/volume1" + Should contain ${result} 200 OK + +Test ozone cli + ${result} = Execute on 1 datanode ozone oz -createVolume o3://ksm/hive -user bilbo -quota 100TB -root + Should contain ${result} Client cannot authenticate via + # Authenticate testuser + Execute on 0 datanode kinit -k testuser/datanode@EXAMPLE.COM -t /etc/security/keytabs/testuser.keytab + Execute on 0 datanode ozone oz -createVolume o3://ksm/hive -user bilbo -quota 100TB -root + ${result} = Execute on 0 datanode ozone oz -listVolume o3://ksm/ -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")' + Should contain ${result} createdOn + Execute on 0 datanode ozone oz -updateVolume o3://ksm/hive -user bill -quota 10TB + ${result} = Execute on 0 datanode ozone oz -infoVolume o3://ksm/hive | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="hive") | .owner | .name' + Should Be Equal ${result} bill + +*** Keywords *** + +Startup Ozone Cluster + ${rc} ${output} = Run docker compose 0 down + ${rc} ${output} = Run docker compose 0 up -d + Should Be Equal As Integers ${rc} 0 + Wait Until Keyword Succeeds 3min 10sec Is Daemon started ksm KSM is listening + +Teardown Ozone Cluster + Run docker compose 0 down + +Is daemon running + [arguments] ${name} + ${result} = Run docker ps + Should contain ${result} _${name}_1 + +Is Daemon started + [arguments] ${name} ${expression} + ${rc} ${result} = Run docker compose 0 logs + Should contain ${result} ${expression} + +Have healthy datanodes + [arguments] ${requirednodes} + ${result} = Execute on 0 scm curl -s 'http://localhost:9876/jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo' | jq -r '.beans[0].NodeCount[] | select(.key=="HEALTHY") | .value' + Should Be Equal ${result} ${requirednodes} + +Execute on + [arguments] ${expected_rc} ${componentname} ${command} + ${rc} ${return} = Run docker compose ${expected_rc} exec ${componentname} ${command} + [return] ${return} + +Run docker compose + [arguments] ${expected_rc} ${command} + Set Environment Variable OZONEDIR ${basedir}/hadoop-dist/target/ozone + ${rc} ${output} = Run And Return Rc And Output docker-compose -f ${basedir}/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml ${command} + Should Be Equal As Integers ${rc} ${expected_rc} + [return] ${rc} ${output} diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java index 9b373aeec5f..4ce22f61606 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java @@ -114,7 +114,7 @@ public class RestClient implements ClientProtocol { try { Preconditions.checkNotNull(conf); this.conf = conf; - + this.ugi = UserGroupInformation.getCurrentUser(); long socketTimeout = conf.getTimeDuration( OzoneConfigKeys.OZONE_CLIENT_SOCKET_TIMEOUT, OzoneConfigKeys.OZONE_CLIENT_SOCKET_TIMEOUT_DEFAULT, @@ -150,7 +150,7 @@ public class RestClient implements ClientProtocol { .setConnectTimeout(Math.toIntExact(connectionTimeout)) .build()) .build(); - this.ugi = UserGroupInformation.getCurrentUser(); + this.userRights = conf.getEnum(OMConfigKeys.OZONE_OM_USER_RIGHTS, OMConfigKeys.OZONE_OM_USER_RIGHTS_DEFAULT); diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index bd80b3ec00f..1f7a5760973 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -76,6 +76,7 @@ import org.apache.ratis.protocol.ClientId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.ws.rs.HEAD; import java.io.IOException; import java.net.InetSocketAddress; import java.util.*; @@ -131,8 +132,7 @@ public class RpcClient implements ClientProtocol { this.ozoneManagerClient = new OzoneManagerProtocolClientSideTranslatorPB( RPC.getProxy(OzoneManagerProtocolPB.class, omVersion, - omAddress, UserGroupInformation.getCurrentUser(), conf, - NetUtils.getDefaultSocketFactory(conf), + omAddress, ugi, conf, NetUtils.getDefaultSocketFactory(conf), Client.getRpcTimeout(conf)), clientId.toString()); long scmVersion = @@ -143,8 +143,7 @@ public class RpcClient implements ClientProtocol { this.storageContainerLocationClient = new StorageContainerLocationProtocolClientSideTranslatorPB( RPC.getProxy(StorageContainerLocationProtocolPB.class, scmVersion, - scmAddress, UserGroupInformation.getCurrentUser(), conf, - NetUtils.getDefaultSocketFactory(conf), + scmAddress, ugi, conf, NetUtils.getDefaultSocketFactory(conf), Client.getRpcTimeout(conf))); this.xceiverClientManager = new XceiverClientManager(conf);