From e47135d9d9759a2eeef99fc8b8df70fcd2f143d6 Mon Sep 17 00:00:00 2001 From: Xiaoyu Yao Date: Tue, 22 May 2018 13:32:28 -0700 Subject: [PATCH] HDDS-70. Fix config names for secure ksm and scm. Contributed by Ajay Kumar. --- .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 6 +- .../protocol/ScmBlockLocationProtocol.java | 2 +- .../StorageContainerLocationProtocol.java | 3 +- .../ScmBlockLocationProtocolPB.java | 4 +- .../StorageContainerLocationProtocolPB.java | 2 +- .../apache/hadoop/ozone/OzoneConfigKeys.java | 1 - .../src/main/resources/ozone-default.xml | 31 +++---- .../StorageContainerDatanodeProtocol.java | 2 +- .../StorageContainerDatanodeProtocolPB.java | 2 +- .../scm/server/StorageContainerManager.java | 18 ++-- .../compose-secure/docker-compose.yaml | 6 +- .../test/compose/compose-secure/docker-config | 12 +-- .../acceptance/ozone-secure.robot | 12 +-- .../ozone/client/protocol/ClientProtocol.java | 2 +- .../hadoop/ozone/ksm/KSMConfigKeys.java | 84 +++++++++++++++++++ .../om/protocol/OzoneManagerProtocol.java | 4 +- .../hadoop/ozone/TestSecureOzoneCluster.java | 21 +++-- .../apache/hadoop/ozone/om/OzoneManager.java | 4 +- 18 files changed, 144 insertions(+), 72 deletions(-) create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index bceebdca699..be41658014d 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -205,9 +205,9 @@ public final class ScmConfigKeys { "ozone.scm.http-address"; public static final String OZONE_SCM_HTTPS_ADDRESS_KEY = "ozone.scm.https-address"; - public static final String OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY = - "ozone.scm.kerberos.keytab.file"; - public static final String OZONE_SCM_KERBEROS_PRINCIPAL_KEY = "ozone.scm.kerberos.principal"; + public static final String HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY = + "hdds.scm.kerberos.keytab.file"; + public static final String HDDS_SCM_KERBEROS_PRINCIPAL_KEY = "hdds.scm.kerberos.principal"; public static final String OZONE_SCM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0"; public static final int OZONE_SCM_HTTP_BIND_PORT_DEFAULT = 9876; public static final int OZONE_SCM_HTTPS_BIND_PORT_DEFAULT = 9877; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java index e17f1c27697..2d46ae03c94 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java @@ -33,7 +33,7 @@ import java.util.List; * ScmBlockLocationProtocol is used by an HDFS node to find the set of nodes * to read/write a block. */ -@KerberosInfo(serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY) +@KerberosInfo(serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY) public interface ScmBlockLocationProtocol { /** diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java index 5bc2521848b..e21bc531e58 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java @@ -17,7 +17,6 @@ package org.apache.hadoop.hdds.scm.protocol; -import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.ScmInfo; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; @@ -35,7 +34,7 @@ import org.apache.hadoop.security.KerberosInfo; * ContainerLocationProtocol is used by an HDFS node to find the set of nodes * that currently host a container. */ -@KerberosInfo(serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY) +@KerberosInfo(serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY) public interface StorageContainerLocationProtocol { /** * Asks SCM where a container should be allocated. SCM responds with the diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java index 89bb066c933..06bbd05e394 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java @@ -18,11 +18,9 @@ package org.apache.hadoop.hdds.scm.protocolPB; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.protocol.proto.ScmBlockLocationProtocolProtos .ScmBlockLocationProtocolService; import org.apache.hadoop.hdds.scm.ScmConfigKeys; -import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.ipc.ProtocolInfo; import org.apache.hadoop.security.KerberosInfo; @@ -35,7 +33,7 @@ import org.apache.hadoop.security.KerberosInfo; protocolVersion = 1) @InterfaceAudience.Private @KerberosInfo( - serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY) + serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY) public interface ScmBlockLocationProtocolPB extends ScmBlockLocationProtocolService.BlockingInterface { } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java index 3bd83f9f7f7..f80ba2010c8 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java @@ -33,7 +33,7 @@ import org.apache.hadoop.security.KerberosInfo; "org.apache.hadoop.ozone.protocol.StorageContainerLocationProtocol", protocolVersion = 1) @KerberosInfo( - serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY) + serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY) @InterfaceAudience.Private public interface StorageContainerLocationProtocolPB extends StorageContainerLocationProtocolService.BlockingInterface { diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index 0f101ac755a..2dc7b97993c 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -352,7 +352,6 @@ public final class OzoneConfigKeys { public static final String OZONE_CONTAINER_COPY_WORKDIR = "hdds.datanode.replication.work.dir"; - /** * Config properties to set client side checksum properties. */ diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index c113c2608b3..d1f9785a5a7 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -529,14 +529,6 @@ ozone.om.http-address. - - ozone.om.keytab.file - - OM, SECURITY - - The keytab file for Kerberos authentication in OM. - - ozone.om.db.cache.size.mb 128 @@ -1049,7 +1041,7 @@ - ozone.scm.container.creation.lease.timeout + hdds.scm.container.creation.lease.timeout 60s OZONE, SCM @@ -1585,15 +1577,18 @@ ozone.scm.kerberos.keytab.file +======= + hdds.scm.kerberos.keytab.file +>>>>>>> HDDS-70. Fix config names for secure ksm and scm. Contributed by Ajay Kumar. OZONE, SECURITY The keytab file used by each SCM daemon to login as its service principal. The principal name is configured with - ozone.scm.kerberos.principal. + hdds.scm.kerberos.principal. - ozone.scm.kerberos.principal + hdds.scm.kerberos.principal OZONE, SECURITY The SCM service principal. Ex scm/_HOST@REALM.COM @@ -1603,24 +1598,24 @@ ozone.om.kerberos.keytab.file OZONE, SECURITY - The keytab file used by KSM daemon to login as its + The keytab file used by OzoneManager daemon to login as its service principal. The principal name is configured with - hdds.ksm.kerberos.principal. + ozone.om.kerberos.principal. ozone.om.kerberos.principal OZONE, SECURITY - The KSM service principal. Ex ksm/_HOST@REALM.COM + The OzoneManager service principal. Ex om/_HOST@REALM.COM - ozone.scm.web.authentication.kerberos.principal + hdds.scm.web.authentication.kerberos.principal HTTP/_HOST@EXAMPLE.COM - ozone.scm.web.authentication.kerberos.keytab + hdds.scm.web.authentication.kerberos.keytab /etc/security/keytabs/HTTP.keytab @@ -1628,14 +1623,14 @@ ozone.om.http.kerberos.principal HTTP/_HOST@EXAMPLE.COM - KSM http server kerberos principal. + OzoneManager http server kerberos principal. ozone.om.http.kerberos.keytab.file /etc/security/keytabs/HTTP.keytab - KSM http server kerberos keytab. + OzoneManager http server kerberos keytab. diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java index 8049e9d2cbe..360058110e1 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java @@ -44,7 +44,7 @@ import org.apache.hadoop.security.KerberosInfo; * Protoc file that defines this protocol. */ @KerberosInfo( - serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY) + serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY) @InterfaceAudience.Private public interface StorageContainerDatanodeProtocol { /** diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java index 9c32ef898a4..9006e9175ac 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java @@ -33,7 +33,7 @@ import org.apache.hadoop.security.KerberosInfo; "org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol", protocolVersion = 1) @KerberosInfo( - serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY, + serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY, clientPrincipal = DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY) public interface StorageContainerDatanodeProtocolPB extends StorageContainerDatanodeProtocolService.BlockingInterface { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 0f295cc681b..4f6558c522d 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -110,8 +110,8 @@ import java.util.concurrent.TimeUnit; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY; import static org.apache.hadoop.util.ExitUtil.terminate; /** @@ -339,17 +339,17 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl private void loginAsSCMUser(Configuration conf) throws IOException, AuthenticationException { LOG.debug("Ozone security is enabled. Attempting login for SCM user. " - + "Principal: {}, keytab: {}", conf.get - (OZONE_SCM_KERBEROS_PRINCIPAL_KEY), - conf.get(OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY)); + + "Principal: {}, keytab: {}", + conf.get(HDDS_SCM_KERBEROS_PRINCIPAL_KEY), + conf.get(HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY)); - if (SecurityUtil.getAuthenticationMethod(conf).equals - (AuthenticationMethod.KERBEROS)) { + if (SecurityUtil.getAuthenticationMethod(conf).equals( + AuthenticationMethod.KERBEROS)) { UserGroupInformation.setConfiguration(conf); InetSocketAddress socAddr = HddsServerUtil .getScmBlockClientBindAddress(conf); - SecurityUtil.login(conf, OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY, - OZONE_SCM_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName()); + SecurityUtil.login(conf, HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY, + HDDS_SCM_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName()); } else { throw new AuthenticationException(SecurityUtil.getAuthenticationMethod( conf) + " authentication method not support. " diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml index 2661163c0d6..db211bc3c55 100644 --- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml +++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml @@ -40,15 +40,15 @@ services: env_file: - ./docker-config command: ["/opt/hadoop/bin/ozone","datanode"] - ksm: + om: image: ahadoop/ozone:v1 - hostname: ksm + hostname: om volumes: - ${OZONEDIR}:/opt/hadoop ports: - 9874:9874 environment: - ENSURE_KSM_INITIALIZED: /data/metadata/ksm/current/VERSION + ENSURE_KSM_INITIALIZED: /data/metadata/om/current/VERSION env_file: - ./docker-config command: ["/opt/hadoop/bin/ozone","ksm"] diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config index 678c75af7c9..360b69a8861 100644 --- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config +++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -OZONE-SITE.XML_ozone.ksm.address=ksm +OZONE-SITE.XML_ozone.ksm.address=om OZONE-SITE.XML_ozone.scm.names=scm OZONE-SITE.XML_ozone.enabled=True OZONE-SITE.XML_hdds.scm.datanode.id=/data/datanode.id @@ -25,13 +25,13 @@ OZONE-SITE.XML_hdds.scm.client.address=scm OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab -OZONE-SITE.XML_ozone.ksm.kerberos.principal=ksm/ksm@EXAMPLE.COM -OZONE-SITE.XML_ozone.ksm.kerberos.keytab.file=/etc/security/keytabs/ksm.keytab +OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM +OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab OZONE-SITE.XML_ozone.security.enabled=true OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.principal=HTTP/scm@EXAMPLE.COM OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab -OZONE-SITE.XML_ozone.ksm.web.authentication.kerberos.principal=HTTP/ksm@EXAMPLE.COM -OZONE-SITE.XML_ozone.ksm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab +OZONE-SITE.XML_ozone.om.web.authentication.kerberos.principal=HTTP/om@EXAMPLE.COM +OZONE-SITE.XML_ozone.om.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.scm.client.address=scm HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode @@ -57,7 +57,7 @@ LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH OZONE_DATANODE_SECURE_USER=root CONF_DIR=/etc/security/keytabs -KERBEROS_KEYTABS=dn nn ksm scm HTTP testuser +KERBEROS_KEYTABS=dn nn om scm HTTP testuser KERBEROS_KEYSTORES=hadoop KERBEROS_SERVER=ozone.kdc JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ diff --git a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot index 4a789804fe9..7fc1088e680 100644 --- a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot +++ b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot @@ -26,7 +26,7 @@ ${version} *** Test Cases *** Daemons are running - Is daemon running ksm + Is daemon running om Is daemon running scm Is daemon running datanode Is daemon running ozone.kdc @@ -45,15 +45,15 @@ Test rest interface Should contain ${result} 200 OK Test ozone cli - ${result} = Execute on 1 datanode ozone oz -createVolume o3://ksm/hive -user bilbo -quota 100TB -root + ${result} = Execute on 1 datanode ozone oz -createVolume o3://om/hive -user bilbo -quota 100TB -root Should contain ${result} Client cannot authenticate via # Authenticate testuser Execute on 0 datanode kinit -k testuser/datanode@EXAMPLE.COM -t /etc/security/keytabs/testuser.keytab - Execute on 0 datanode ozone oz -createVolume o3://ksm/hive -user bilbo -quota 100TB -root - ${result} = Execute on 0 datanode ozone oz -listVolume o3://ksm/ -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")' + Execute on 0 datanode ozone oz -createVolume o3://om/hive -user bilbo -quota 100TB -root + ${result} = Execute on 0 datanode ozone oz -listVolume o3://om/ -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")' Should contain ${result} createdOn - Execute on 0 datanode ozone oz -updateVolume o3://ksm/hive -user bill -quota 10TB - ${result} = Execute on 0 datanode ozone oz -infoVolume o3://ksm/hive | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="hive") | .owner | .name' + Execute on 0 datanode ozone oz -updateVolume o3://om/hive -user bill -quota 10TB + ${result} = Execute on 0 datanode ozone oz -infoVolume o3://om/hive | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="hive") | .owner | .name' Should Be Equal ${result} bill *** Keywords *** diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index ed4dcef692c..3c586305f8b 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -43,7 +43,7 @@ import org.apache.hadoop.security.KerberosInfo; * includes: {@link org.apache.hadoop.ozone.client.rpc.RpcClient} for RPC and * {@link org.apache.hadoop.ozone.client.rest.RestClient} for REST. */ -@KerberosInfo(serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY) +@KerberosInfo(serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY) public interface ClientProtocol { /** diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java new file mode 100644 index 00000000000..cc25dbe0170 --- /dev/null +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.ksm; + +import org.apache.hadoop.ozone.OzoneAcl; +/** + * KSM Constants. + */ +public final class KSMConfigKeys { + /** + * Never constructed. + */ + private KSMConfigKeys() { + } + + + public static final String OZONE_KSM_HANDLER_COUNT_KEY = + "ozone.ksm.handler.count.key"; + public static final int OZONE_KSM_HANDLER_COUNT_DEFAULT = 20; + + public static final String OZONE_KSM_ADDRESS_KEY = + "ozone.ksm.address"; + public static final String OZONE_KSM_BIND_HOST_DEFAULT = + "0.0.0.0"; + public static final int OZONE_KSM_PORT_DEFAULT = 9862; + + public static final String OZONE_KSM_HTTP_ENABLED_KEY = + "ozone.ksm.http.enabled"; + public static final String OZONE_KSM_HTTP_BIND_HOST_KEY = + "ozone.ksm.http-bind-host"; + public static final String OZONE_KSM_HTTPS_BIND_HOST_KEY = + "ozone.ksm.https-bind-host"; + public static final String OZONE_KSM_HTTP_ADDRESS_KEY = + "ozone.ksm.http-address"; + public static final String OZONE_KSM_HTTPS_ADDRESS_KEY = + "ozone.ksm.https-address"; + public static final String OZONE_KSM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0"; + public static final int OZONE_KSM_HTTP_BIND_PORT_DEFAULT = 9874; + public static final int OZONE_KSM_HTTPS_BIND_PORT_DEFAULT = 9875; + + // LevelDB cache file uses an off-heap cache in LevelDB of 128 MB. + public static final String OZONE_KSM_DB_CACHE_SIZE_MB = + "ozone.ksm.db.cache.size.mb"; + public static final int OZONE_KSM_DB_CACHE_SIZE_DEFAULT = 128; + + public static final String OZONE_KSM_USER_MAX_VOLUME = + "ozone.ksm.user.max.volume"; + public static final int OZONE_KSM_USER_MAX_VOLUME_DEFAULT = 1024; + + // KSM Default user/group permissions + public static final String OZONE_KSM_USER_RIGHTS = + "ozone.ksm.user.rights"; + public static final OzoneAcl.OzoneACLRights OZONE_KSM_USER_RIGHTS_DEFAULT = + OzoneAcl.OzoneACLRights.READ_WRITE; + + public static final String OZONE_KSM_GROUP_RIGHTS = + "ozone.ksm.group.rights"; + public static final OzoneAcl.OzoneACLRights OZONE_KSM_GROUP_RIGHTS_DEFAULT = + OzoneAcl.OzoneACLRights.READ_WRITE; + + public static final String OZONE_KEY_DELETING_LIMIT_PER_TASK = + "ozone.key.deleting.limit.per.task"; + public static final int OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT = 1000; + + public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL = + "ozone.om.web.authentication.kerberos.principal"; + public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE = + "ozone.om.web.authentication.kerberos.keytab"; +} diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index 2e007ac1b69..e4bfc839211 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -30,10 +30,8 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadList; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.helpers.ServiceInfo; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo; - -import org.apache.hadoop.ozone.protocol.proto - .OzoneManagerProtocolProtos.OzoneAclInfo; import java.io.IOException; import java.util.List; import org.apache.hadoop.security.KerberosInfo; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java index 0952cbd6c3e..2fe00e3bf27 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java @@ -120,10 +120,10 @@ public final class TestSecureOzoneCluster { private void createCredentialsInKDC(Configuration conf, MiniKdc miniKdc) throws Exception { createPrincipal(scmKeytab, - conf.get(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)); - createPrincipal(spnegoKeytab, - conf.get(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY), - conf.get(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY)); + conf.get(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)); + createPrincipal(spnegoKeytab, + conf.get(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY), + conf.get(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY)); createPrincipal(omKeyTab, conf.get(OMConfigKeys .OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY)); @@ -156,7 +156,7 @@ public final class TestSecureOzoneCluster { "kerberos"); conf.set(OZONE_ADMINISTRATORS, curUser); - conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY, + conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY, "scm/" + host + "@" + realm); conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY, "HTTP_SCM/" + host + "@" + realm); @@ -164,19 +164,18 @@ public final class TestSecureOzoneCluster { conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY, "om/" + host + "@" + realm); conf.set(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY, - "HTTP_KSM/" + host + "@" + realm); + "HTTP_OM/" + host + "@" + realm); scmKeytab = new File(workDir, "scm.keytab"); spnegoKeytab = new File(workDir, "http.keytab"); omKeyTab = new File(workDir, "om.keytab"); - conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY, + conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY, scmKeytab.getAbsolutePath()); conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY, spnegoKeytab.getAbsolutePath()); conf.set(OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY, omKeyTab.getAbsolutePath()); - } @Test @@ -207,7 +206,7 @@ public final class TestSecureOzoneCluster { @Test public void testSecureScmStartupFailure() throws Exception { initSCM(); - conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY, ""); + conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY, ""); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); @@ -217,9 +216,9 @@ public final class TestSecureOzoneCluster { StorageContainerManager.createSCM(null, conf); }); - conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY, + conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY, "scm/_HOST@EXAMPLE.com"); - conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY, + conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY, "/etc/security/keytabs/scm.keytab"); testCommonKerberosFailures( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index f0e5a6f852e..6578cfee4d7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -325,8 +325,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl if (SecurityUtil.getAuthenticationMethod(conf).equals (AuthenticationMethod.KERBEROS)) { LOG.debug("Ozone security is enabled. Attempting login for KSM user. " - + "Principal: {},keytab: {}", conf.get - (OZONE_OM_KERBEROS_PRINCIPAL_KEY), + + "Principal: {},keytab: {}", conf.get( + OZONE_OM_KERBEROS_PRINCIPAL_KEY), conf.get(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY)); UserGroupInformation.setConfiguration(conf);