Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDDS-70. Fix config names for secure ksm and scm. Contributed by Ajay Kumar.

This commit is contained in:
Xiaoyu Yao 2018-05-22 13:32:28 -07:00
parent 914e93beeb
commit e47135d9d9
18 changed files with 144 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
View File

@ -205,9 +205,9 @@ public final class ScmConfigKeys {
"ozone.scm.http-address"; "ozone.scm.http-address";
public static final String OZONE_SCM_HTTPS_ADDRESS_KEY = public static final String OZONE_SCM_HTTPS_ADDRESS_KEY =
"ozone.scm.https-address"; "ozone.scm.https-address";
public static final String OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY = public static final String HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY =
"ozone.scm.kerberos.keytab.file"; "hdds.scm.kerberos.keytab.file";
public static final String OZONE_SCM_KERBEROS_PRINCIPAL_KEY = "ozone.scm.kerberos.principal"; public static final String HDDS_SCM_KERBEROS_PRINCIPAL_KEY = "hdds.scm.kerberos.principal";
public static final String OZONE_SCM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0"; public static final String OZONE_SCM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
public static final int OZONE_SCM_HTTP_BIND_PORT_DEFAULT = 9876; public static final int OZONE_SCM_HTTP_BIND_PORT_DEFAULT = 9876;
public static final int OZONE_SCM_HTTPS_BIND_PORT_DEFAULT = 9877; public static final int OZONE_SCM_HTTPS_BIND_PORT_DEFAULT = 9877;

2
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
View File

@ -33,7 +33,7 @@ import java.util.List;
* ScmBlockLocationProtocol is used by an HDFS node to find the set of nodes * ScmBlockLocationProtocol is used by an HDFS node to find the set of nodes
* to read/write a block. * to read/write a block.
*/ */
@KerberosInfo(serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY) @KerberosInfo(serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
public interface ScmBlockLocationProtocol { public interface ScmBlockLocationProtocol {
/** /**

3
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
View File

@ -17,7 +17,6 @@
package org.apache.hadoop.hdds.scm.protocol; package org.apache.hadoop.hdds.scm.protocol;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.ScmInfo; import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
@ -35,7 +34,7 @@ import org.apache.hadoop.security.KerberosInfo;
* ContainerLocationProtocol is used by an HDFS node to find the set of nodes * ContainerLocationProtocol is used by an HDFS node to find the set of nodes
* that currently host a container. * that currently host a container.
*/ */
@KerberosInfo(serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY) @KerberosInfo(serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
public interface StorageContainerLocationProtocol { public interface StorageContainerLocationProtocol {
/** /**
* Asks SCM where a container should be allocated. SCM responds with the * Asks SCM where a container should be allocated. SCM responds with the

4
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java
View File

@ -18,11 +18,9 @@
package org.apache.hadoop.hdds.scm.protocolPB; package org.apache.hadoop.hdds.scm.protocolPB;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.protocol.proto.ScmBlockLocationProtocolProtos import org.apache.hadoop.hdds.protocol.proto.ScmBlockLocationProtocolProtos
.ScmBlockLocationProtocolService; .ScmBlockLocationProtocolService;
import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.ipc.ProtocolInfo; import org.apache.hadoop.ipc.ProtocolInfo;
import org.apache.hadoop.security.KerberosInfo; import org.apache.hadoop.security.KerberosInfo;
@ -35,7 +33,7 @@ import org.apache.hadoop.security.KerberosInfo;
protocolVersion = 1) protocolVersion = 1)
@InterfaceAudience.Private @InterfaceAudience.Private
@KerberosInfo( @KerberosInfo(
serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY) serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
public interface ScmBlockLocationProtocolPB public interface ScmBlockLocationProtocolPB
extends ScmBlockLocationProtocolService.BlockingInterface { extends ScmBlockLocationProtocolService.BlockingInterface {
} }

2
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java
View File

@ -33,7 +33,7 @@ import org.apache.hadoop.security.KerberosInfo;
"org.apache.hadoop.ozone.protocol.StorageContainerLocationProtocol", "org.apache.hadoop.ozone.protocol.StorageContainerLocationProtocol",
protocolVersion = 1) protocolVersion = 1)
@KerberosInfo( @KerberosInfo(
serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY) serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
@InterfaceAudience.Private @InterfaceAudience.Private
public interface StorageContainerLocationProtocolPB public interface StorageContainerLocationProtocolPB
extends StorageContainerLocationProtocolService.BlockingInterface { extends StorageContainerLocationProtocolService.BlockingInterface {

1
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
View File

@ -352,7 +352,6 @@ public final class OzoneConfigKeys {
public static final String OZONE_CONTAINER_COPY_WORKDIR = public static final String OZONE_CONTAINER_COPY_WORKDIR =
"hdds.datanode.replication.work.dir"; "hdds.datanode.replication.work.dir";
/** /**
* Config properties to set client side checksum properties. * Config properties to set client side checksum properties.
*/ */

31
hadoop-hdds/common/src/main/resources/ozone-default.xml
View File

@ -529,14 +529,6 @@
ozone.om.http-address. ozone.om.http-address.
</description> </description>
</property> </property>
<property>
<name>ozone.om.keytab.file</name>
<value/>
<tag>OM, SECURITY</tag>
<description>
The keytab file for Kerberos authentication in OM.
</description>
</property>
<property> <property>
<name>ozone.om.db.cache.size.mb</name> <name>ozone.om.db.cache.size.mb</name>
<value>128</value> <value>128</value>
@ -1049,7 +1041,7 @@
</property> </property>
<property> <property>
<name>ozone.scm.container.creation.lease.timeout</name> <name>hdds.scm.container.creation.lease.timeout</name>
<value>60s</value> <value>60s</value>
<tag>OZONE, SCM</tag> <tag>OZONE, SCM</tag>
<description> <description>
@ -1585,15 +1577,18 @@
</property> </property>
<property> <property>
<name>ozone.scm.kerberos.keytab.file</name> <name>ozone.scm.kerberos.keytab.file</name>
=======
<name>hdds.scm.kerberos.keytab.file</name>
>>>>>>> HDDS-70. Fix config names for secure ksm and scm. Contributed by Ajay Kumar.
<value></value> <value></value>
<tag> OZONE, SECURITY</tag> <tag> OZONE, SECURITY</tag>
<description> The keytab file used by each SCM daemon to login as its <description> The keytab file used by each SCM daemon to login as its
service principal. The principal name is configured with service principal. The principal name is configured with
ozone.scm.kerberos.principal. hdds.scm.kerberos.principal.
</description> </description>
</property> </property>
<property> <property>
<name>ozone.scm.kerberos.principal</name> <name>hdds.scm.kerberos.principal</name>
<value></value> <value></value>
<tag> OZONE, SECURITY</tag> <tag> OZONE, SECURITY</tag>
<description>The SCM service principal. Ex scm/_HOST@REALM.COM</description> <description>The SCM service principal. Ex scm/_HOST@REALM.COM</description>
@ -1603,24 +1598,24 @@
<name>ozone.om.kerberos.keytab.file</name> <name>ozone.om.kerberos.keytab.file</name>
<value></value> <value></value>
<tag> OZONE, SECURITY</tag> <tag> OZONE, SECURITY</tag>
<description> The keytab file used by KSM daemon to login as its <description> The keytab file used by OzoneManager daemon to login as its
service principal. The principal name is configured with service principal. The principal name is configured with
hdds.ksm.kerberos.principal. ozone.om.kerberos.principal.
</description> </description>
</property> </property>
<property> <property>
<name>ozone.om.kerberos.principal</name> <name>ozone.om.kerberos.principal</name>
<value></value> <value></value>
<tag> OZONE, SECURITY</tag> <tag> OZONE, SECURITY</tag>
<description>The KSM service principal. Ex ksm/_HOST@REALM.COM</description> <description>The OzoneManager service principal. Ex om/_HOST@REALM.COM</description>
</property> </property>
<property> <property>
<name>ozone.scm.web.authentication.kerberos.principal</name> <name>hdds.scm.web.authentication.kerberos.principal</name>
<value>HTTP/_HOST@EXAMPLE.COM</value> <value>HTTP/_HOST@EXAMPLE.COM</value>
</property> </property>
<property> <property>
<name>ozone.scm.web.authentication.kerberos.keytab</name> <name>hdds.scm.web.authentication.kerberos.keytab</name>
<value>/etc/security/keytabs/HTTP.keytab</value> <value>/etc/security/keytabs/HTTP.keytab</value>
</property> </property>
@ -1628,14 +1623,14 @@
<name>ozone.om.http.kerberos.principal</name> <name>ozone.om.http.kerberos.principal</name>
<value>HTTP/_HOST@EXAMPLE.COM</value> <value>HTTP/_HOST@EXAMPLE.COM</value>
<description> <description>
KSM http server kerberos principal. OzoneManager http server kerberos principal.
</description> </description>
</property> </property>
<property> <property>
<name>ozone.om.http.kerberos.keytab.file</name> <name>ozone.om.http.kerberos.keytab.file</name>
<value>/etc/security/keytabs/HTTP.keytab</value> <value>/etc/security/keytabs/HTTP.keytab</value>
<description> <description>
KSM http server kerberos keytab. OzoneManager http server kerberos keytab.
</description> </description>
</property> </property>
</configuration> </configuration>

2
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
View File

@ -44,7 +44,7 @@ import org.apache.hadoop.security.KerberosInfo;
* Protoc file that defines this protocol. * Protoc file that defines this protocol.
*/ */
@KerberosInfo( @KerberosInfo(
serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY) serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
@InterfaceAudience.Private @InterfaceAudience.Private
public interface StorageContainerDatanodeProtocol { public interface StorageContainerDatanodeProtocol {
/** /**

2
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
View File

@ -33,7 +33,7 @@ import org.apache.hadoop.security.KerberosInfo;
"org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol", "org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol",
protocolVersion = 1) protocolVersion = 1)
@KerberosInfo( @KerberosInfo(
serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY, serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY,
clientPrincipal = DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY) clientPrincipal = DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY)
public interface StorageContainerDatanodeProtocolPB extends public interface StorageContainerDatanodeProtocolPB extends
StorageContainerDatanodeProtocolService.BlockingInterface { StorageContainerDatanodeProtocolService.BlockingInterface {

18
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
View File

@ -110,8 +110,8 @@ import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY;
import static org.apache.hadoop.util.ExitUtil.terminate; import static org.apache.hadoop.util.ExitUtil.terminate;
/** /**
@ -339,17 +339,17 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
private void loginAsSCMUser(Configuration conf) private void loginAsSCMUser(Configuration conf)
throws IOException, AuthenticationException { throws IOException, AuthenticationException {
LOG.debug("Ozone security is enabled. Attempting login for SCM user. " LOG.debug("Ozone security is enabled. Attempting login for SCM user. "
+ "Principal: {}, keytab: {}", conf.get + "Principal: {}, keytab: {}",
(OZONE_SCM_KERBEROS_PRINCIPAL_KEY), conf.get(HDDS_SCM_KERBEROS_PRINCIPAL_KEY),
conf.get(OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY)); conf.get(HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY));
if (SecurityUtil.getAuthenticationMethod(conf).equals if (SecurityUtil.getAuthenticationMethod(conf).equals(
(AuthenticationMethod.KERBEROS)) { AuthenticationMethod.KERBEROS)) {
UserGroupInformation.setConfiguration(conf); UserGroupInformation.setConfiguration(conf);
InetSocketAddress socAddr = HddsServerUtil InetSocketAddress socAddr = HddsServerUtil
.getScmBlockClientBindAddress(conf); .getScmBlockClientBindAddress(conf);
SecurityUtil.login(conf, OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY, SecurityUtil.login(conf, HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY,
OZONE_SCM_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName()); HDDS_SCM_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName());
} else { } else {
throw new AuthenticationException(SecurityUtil.getAuthenticationMethod( throw new AuthenticationException(SecurityUtil.getAuthenticationMethod(
conf) + " authentication method not support. " conf) + " authentication method not support. "

6
hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
View File

@ -40,15 +40,15 @@ services:
env_file: env_file:
- ./docker-config - ./docker-config
command: ["/opt/hadoop/bin/ozone","datanode"] command: ["/opt/hadoop/bin/ozone","datanode"]
ksm: om:
image: ahadoop/ozone:v1 image: ahadoop/ozone:v1
hostname: ksm hostname: om
volumes: volumes:
- ${OZONEDIR}:/opt/hadoop - ${OZONEDIR}:/opt/hadoop
ports: ports:
- 9874:9874 - 9874:9874
environment: environment:
ENSURE_KSM_INITIALIZED: /data/metadata/ksm/current/VERSION ENSURE_KSM_INITIALIZED: /data/metadata/om/current/VERSION
env_file: env_file:
- ./docker-config - ./docker-config
command: ["/opt/hadoop/bin/ozone","ksm"] command: ["/opt/hadoop/bin/ozone","ksm"]

12
hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
View File

@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
OZONE-SITE.XML_ozone.ksm.address=ksm OZONE-SITE.XML_ozone.ksm.address=om
OZONE-SITE.XML_ozone.scm.names=scm OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.enabled=True OZONE-SITE.XML_ozone.enabled=True
OZONE-SITE.XML_hdds.scm.datanode.id=/data/datanode.id OZONE-SITE.XML_hdds.scm.datanode.id=/data/datanode.id
@ -25,13 +25,13 @@ OZONE-SITE.XML_hdds.scm.client.address=scm
OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
OZONE-SITE.XML_ozone.ksm.kerberos.principal=ksm/ksm@EXAMPLE.COM OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
OZONE-SITE.XML_ozone.ksm.kerberos.keytab.file=/etc/security/keytabs/ksm.keytab OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
OZONE-SITE.XML_ozone.security.enabled=true OZONE-SITE.XML_ozone.security.enabled=true
OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.principal=HTTP/scm@EXAMPLE.COM OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.principal=HTTP/scm@EXAMPLE.COM
OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
OZONE-SITE.XML_ozone.ksm.web.authentication.kerberos.principal=HTTP/ksm@EXAMPLE.COM OZONE-SITE.XML_ozone.om.web.authentication.kerberos.principal=HTTP/om@EXAMPLE.COM
OZONE-SITE.XML_ozone.ksm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab OZONE-SITE.XML_ozone.om.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_ozone.scm.client.address=scm
HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
@ -57,7 +57,7 @@ LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH
OZONE_DATANODE_SECURE_USER=root OZONE_DATANODE_SECURE_USER=root
CONF_DIR=/etc/security/keytabs CONF_DIR=/etc/security/keytabs
KERBEROS_KEYTABS=dn nn ksm scm HTTP testuser KERBEROS_KEYTABS=dn nn om scm HTTP testuser
KERBEROS_KEYSTORES=hadoop KERBEROS_KEYSTORES=hadoop
KERBEROS_SERVER=ozone.kdc KERBEROS_SERVER=ozone.kdc
JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/

12
hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
View File

@ -26,7 +26,7 @@ ${version}
*** Test Cases *** *** Test Cases ***
Daemons are running Daemons are running
Is daemon running ksm Is daemon running om
Is daemon running scm Is daemon running scm
Is daemon running datanode Is daemon running datanode
Is daemon running ozone.kdc Is daemon running ozone.kdc
@ -45,15 +45,15 @@ Test rest interface
Should contain ${result} 200 OK Should contain ${result} 200 OK
Test ozone cli Test ozone cli
${result} = Execute on 1 datanode ozone oz -createVolume o3://ksm/hive -user bilbo -quota 100TB -root ${result} = Execute on 1 datanode ozone oz -createVolume o3://om/hive -user bilbo -quota 100TB -root
Should contain ${result} Client cannot authenticate via Should contain ${result} Client cannot authenticate via
# Authenticate testuser # Authenticate testuser
Execute on 0 datanode kinit -k testuser/datanode@EXAMPLE.COM -t /etc/security/keytabs/testuser.keytab Execute on 0 datanode kinit -k testuser/datanode@EXAMPLE.COM -t /etc/security/keytabs/testuser.keytab
Execute on 0 datanode ozone oz -createVolume o3://ksm/hive -user bilbo -quota 100TB -root Execute on 0 datanode ozone oz -createVolume o3://om/hive -user bilbo -quota 100TB -root
${result} = Execute on 0 datanode ozone oz -listVolume o3://ksm/ -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")' ${result} = Execute on 0 datanode ozone oz -listVolume o3://om/ -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")'
Should contain ${result} createdOn Should contain ${result} createdOn
Execute on 0 datanode ozone oz -updateVolume o3://ksm/hive -user bill -quota 10TB Execute on 0 datanode ozone oz -updateVolume o3://om/hive -user bill -quota 10TB
${result} = Execute on 0 datanode ozone oz -infoVolume o3://ksm/hive | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="hive") | .owner | .name' ${result} = Execute on 0 datanode ozone oz -infoVolume o3://om/hive | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="hive") | .owner | .name'
Should Be Equal ${result} bill Should Be Equal ${result} bill
*** Keywords *** *** Keywords ***

2
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
View File

@ -43,7 +43,7 @@ import org.apache.hadoop.security.KerberosInfo;
* includes: {@link org.apache.hadoop.ozone.client.rpc.RpcClient} for RPC and * includes: {@link org.apache.hadoop.ozone.client.rpc.RpcClient} for RPC and
* {@link org.apache.hadoop.ozone.client.rest.RestClient} for REST. * {@link org.apache.hadoop.ozone.client.rest.RestClient} for REST.
*/ */
@KerberosInfo(serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY) @KerberosInfo(serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
public interface ClientProtocol { public interface ClientProtocol {
/** /**

84
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java Normal file
View File

@ -0,0 +1,84 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.ozone.ksm;
import org.apache.hadoop.ozone.OzoneAcl;
/**
* KSM Constants.
*/
public final class KSMConfigKeys {
/**
* Never constructed.
*/
private KSMConfigKeys() {
}
public static final String OZONE_KSM_HANDLER_COUNT_KEY =
"ozone.ksm.handler.count.key";
public static final int OZONE_KSM_HANDLER_COUNT_DEFAULT = 20;
public static final String OZONE_KSM_ADDRESS_KEY =
"ozone.ksm.address";
public static final String OZONE_KSM_BIND_HOST_DEFAULT =
"0.0.0.0";
public static final int OZONE_KSM_PORT_DEFAULT = 9862;
public static final String OZONE_KSM_HTTP_ENABLED_KEY =
"ozone.ksm.http.enabled";
public static final String OZONE_KSM_HTTP_BIND_HOST_KEY =
"ozone.ksm.http-bind-host";
public static final String OZONE_KSM_HTTPS_BIND_HOST_KEY =
"ozone.ksm.https-bind-host";
public static final String OZONE_KSM_HTTP_ADDRESS_KEY =
"ozone.ksm.http-address";
public static final String OZONE_KSM_HTTPS_ADDRESS_KEY =
"ozone.ksm.https-address";
public static final String OZONE_KSM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
public static final int OZONE_KSM_HTTP_BIND_PORT_DEFAULT = 9874;
public static final int OZONE_KSM_HTTPS_BIND_PORT_DEFAULT = 9875;
// LevelDB cache file uses an off-heap cache in LevelDB of 128 MB.
public static final String OZONE_KSM_DB_CACHE_SIZE_MB =
"ozone.ksm.db.cache.size.mb";
public static final int OZONE_KSM_DB_CACHE_SIZE_DEFAULT = 128;
public static final String OZONE_KSM_USER_MAX_VOLUME =
"ozone.ksm.user.max.volume";
public static final int OZONE_KSM_USER_MAX_VOLUME_DEFAULT = 1024;
// KSM Default user/group permissions
public static final String OZONE_KSM_USER_RIGHTS =
"ozone.ksm.user.rights";
public static final OzoneAcl.OzoneACLRights OZONE_KSM_USER_RIGHTS_DEFAULT =
OzoneAcl.OzoneACLRights.READ_WRITE;
public static final String OZONE_KSM_GROUP_RIGHTS =
"ozone.ksm.group.rights";
public static final OzoneAcl.OzoneACLRights OZONE_KSM_GROUP_RIGHTS_DEFAULT =
OzoneAcl.OzoneACLRights.READ_WRITE;
public static final String OZONE_KEY_DELETING_LIMIT_PER_TASK =
"ozone.key.deleting.limit.per.task";
public static final int OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT = 1000;
public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL =
"ozone.om.web.authentication.kerberos.principal";
public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE =
"ozone.om.web.authentication.kerberos.keytab";
}

4
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
View File

@ -30,10 +30,8 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadList;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo; import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo;
import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.OzoneAclInfo;
import java.io.IOException; import java.io.IOException;
import java.util.List; import java.util.List;
import org.apache.hadoop.security.KerberosInfo; import org.apache.hadoop.security.KerberosInfo;

15
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
View File

@ -120,7 +120,7 @@ public final class TestSecureOzoneCluster {
private void createCredentialsInKDC(Configuration conf, MiniKdc miniKdc) private void createCredentialsInKDC(Configuration conf, MiniKdc miniKdc)
throws Exception { throws Exception {
createPrincipal(scmKeytab, createPrincipal(scmKeytab,
conf.get(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)); conf.get(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY));
createPrincipal(spnegoKeytab, createPrincipal(spnegoKeytab,
conf.get(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY), conf.get(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY),
conf.get(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY)); conf.get(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
@ -156,7 +156,7 @@ public final class TestSecureOzoneCluster {
"kerberos"); "kerberos");
conf.set(OZONE_ADMINISTRATORS, curUser); conf.set(OZONE_ADMINISTRATORS, curUser);
conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY, conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY,
"scm/" + host + "@" + realm); "scm/" + host + "@" + realm);
conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY, conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
"HTTP_SCM/" + host + "@" + realm); "HTTP_SCM/" + host + "@" + realm);
@ -164,19 +164,18 @@ public final class TestSecureOzoneCluster {
conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY, conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY,
"om/" + host + "@" + realm); "om/" + host + "@" + realm);
conf.set(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY, conf.set(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
"HTTP_KSM/" + host + "@" + realm); "HTTP_OM/" + host + "@" + realm);
scmKeytab = new File(workDir, "scm.keytab"); scmKeytab = new File(workDir, "scm.keytab");
spnegoKeytab = new File(workDir, "http.keytab"); spnegoKeytab = new File(workDir, "http.keytab");
omKeyTab = new File(workDir, "om.keytab"); omKeyTab = new File(workDir, "om.keytab");
conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY, conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY,
scmKeytab.getAbsolutePath()); scmKeytab.getAbsolutePath());
conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY, conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY,
spnegoKeytab.getAbsolutePath()); spnegoKeytab.getAbsolutePath());
conf.set(OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY, conf.set(OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY,
omKeyTab.getAbsolutePath()); omKeyTab.getAbsolutePath());
} }
@Test @Test
@ -207,7 +206,7 @@ public final class TestSecureOzoneCluster {
@Test @Test
public void testSecureScmStartupFailure() throws Exception { public void testSecureScmStartupFailure() throws Exception {
initSCM(); initSCM();
conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY, ""); conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY, "");
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"kerberos"); "kerberos");
@ -217,9 +216,9 @@ public final class TestSecureOzoneCluster {
StorageContainerManager.createSCM(null, conf); StorageContainerManager.createSCM(null, conf);
}); });
conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY, conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY,
"scm/_HOST@EXAMPLE.com"); "scm/_HOST@EXAMPLE.com");
conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY, conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY,
"/etc/security/keytabs/scm.keytab"); "/etc/security/keytabs/scm.keytab");
testCommonKerberosFailures( testCommonKerberosFailures(

4
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
View File

@ -325,8 +325,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
if (SecurityUtil.getAuthenticationMethod(conf).equals if (SecurityUtil.getAuthenticationMethod(conf).equals
(AuthenticationMethod.KERBEROS)) { (AuthenticationMethod.KERBEROS)) {
LOG.debug("Ozone security is enabled. Attempting login for KSM user. " LOG.debug("Ozone security is enabled. Attempting login for KSM user. "
+ "Principal: {},keytab: {}", conf.get + "Principal: {},keytab: {}", conf.get(
(OZONE_OM_KERBEROS_PRINCIPAL_KEY), OZONE_OM_KERBEROS_PRINCIPAL_KEY),
conf.get(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY)); conf.get(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY));
UserGroupInformation.setConfiguration(conf); UserGroupInformation.setConfiguration(conf);