Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-16350. Ability to tell HDFS client not to request KMS Information from NameNode. Ccontributed by Greg Senia, Ajay Kumar.

This commit is contained in:
Ajay Kumar 2019-06-24 11:38:43 -07:00
parent 004207996c
commit 95c94dcca7
No known key found for this signature in database
GPG Key ID: 8D133AAB81A9B4DD
4 changed files with 52 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
View File

@ -403,4 +403,17 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
public static final Class<? extends DomainNameResolver>
HADOOP_DOMAINNAME_RESOLVER_IMPL_DEFAULT =
DNSDomainNameResolver.class;
/*
* Ignore KMS default URI returned from NameNode.
* When set to true, kms uri is searched in the following order:
* 1. If there is a mapping in Credential's secrets map for namenode uri.
* 2. Fallback to local conf.
* If client choose to ignore KMS uri provided by NameNode then client
* should set KMS URI using 'hadoop.security.key.provider.path' to access
* the right KMS for encrypted files.
* */
public static final String DFS_CLIENT_IGNORE_NAMENODE_DEFAULT_KMS_URI =
"dfs.client.ignore.namenode.default.kms.uri";
public static final boolean
DFS_CLIENT_IGNORE_NAMENODE_DEFAULT_KMS_URI_DEFAULT = false;
}

14
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -3480,4 +3480,18 @@
with the input domain name of the services by querying the underlying DNS.
</description>
</property>
<property>
<name>dfs.client.ignore.namenode.default.kms.uri</name>
<value>false</value>
<description>
Ignore KMS default URI returned from NameNode.
When set to true, kms uri is searched in the following order:
1. If there is a mapping in Credential's secrets map for namenode uri.
2. Fallback to local conf. (i.e hadoop.security.key.provider.path)
If client choose to ignore KMS uri provided by NameNode then client
should set KMS URI using 'hadoop.security.key.provider.path' to access
the right KMS for encrypted files.
</description>
</property>
</configuration>

18
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java
View File

@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hdfs;
import static org.apache.hadoop.fs.CommonConfigurationKeys.DFS_CLIENT_IGNORE_NAMENODE_DEFAULT_KMS_URI;
import static org.apache.hadoop.fs.CommonConfigurationKeys.DFS_CLIENT_IGNORE_NAMENODE_DEFAULT_KMS_URI_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX;
import java.io.IOException;
@ -141,12 +143,20 @@ public final class HdfsKMSUtil {
URI.create(DFSUtilClient.bytes2String(keyProviderUriBytes));
}
if (keyProviderUri == null) {
// NN is old and doesn't report provider, so use conf.
if (keyProviderUriStr == null) {
keyProviderUri = KMSUtil.getKeyProviderUri(conf, keyProviderUriKeyName);
} else if (!keyProviderUriStr.isEmpty()) {
// Check if NN provided uri is not null and ignore property is false.
if (keyProviderUriStr != null && !conf.getBoolean(
DFS_CLIENT_IGNORE_NAMENODE_DEFAULT_KMS_URI,
DFS_CLIENT_IGNORE_NAMENODE_DEFAULT_KMS_URI_DEFAULT)) {
if (!keyProviderUriStr.isEmpty()) {
keyProviderUri = URI.create(keyProviderUriStr);
}
}
// Fallback to configuration.
if (keyProviderUri == null) {
// Either NN is old and doesn't report provider or ignore NN KMS
// provider property is set to true, so use conf.
keyProviderUri = KMSUtil.getKeyProviderUri(conf, keyProviderUriKeyName);
}
if (keyProviderUri != null) {
credentials.addSecretKey(
credsKey, DFSUtilClient.string2Bytes(keyProviderUri.toString()));

11
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
View File

@ -111,6 +111,7 @@ import org.junit.Test;
import org.junit.rules.Timeout;
import org.mockito.Mockito;
import static org.apache.hadoop.fs.CommonConfigurationKeys.DFS_CLIENT_IGNORE_NAMENODE_DEFAULT_KMS_URI;
import static org.junit.Assert.assertNotNull;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
@ -1950,6 +1951,16 @@ public class TestEncryptionZones {
Assert.assertEquals("Key Provider for client and namenode are different",
namenodeKeyProviderUri, cluster.getFileSystem().getClient()
.getKeyProviderUri());
// Ignore the key provider from NN.
clusterConf.setBoolean(
DFS_CLIENT_IGNORE_NAMENODE_DEFAULT_KMS_URI, true);
Assert.assertEquals("Expecting Key Provider for client config",
"dummy://foo:bar@test_provider1", cluster.getFileSystem().getClient()
.getKeyProviderUri().toString());
Assert.assertNotEquals("Key Provider for client and namenode is different",
namenodeKeyProviderUri, cluster.getFileSystem().getClient()
.getKeyProviderUri().toString());
}
/**