From 7d402692b4b06af0459ea81129e93a5182291a27 Mon Sep 17 00:00:00 2001 From: Zhe Zhang Date: Thu, 11 Aug 2016 10:53:16 -0700 Subject: [PATCH] HDFS-9804. Allow long-running Balancer to login with keytab. Contributed by Xiao Chen. (cherry picked from commit ccff6035f50769eb69701128ae61efc69e82609d) --- .../hadoop/security/UserGroupInformation.java | 2 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 6 + .../hadoop/hdfs/server/balancer/Balancer.java | 22 +++- .../src/main/resources/hdfs-default.xml | 46 ++++++- .../hdfs/server/balancer/TestBalancer.java | 123 +++++++++++++++++- 5 files changed, 190 insertions(+), 9 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java index e12c9a3ef6d..389e94a0888 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java @@ -1263,7 +1263,7 @@ public class UserGroupInformation { if (now - user.getLastLogin() < MIN_TIME_BEFORE_RELOGIN ) { LOG.warn("Not attempting to re-login since the last re-login was " + "attempted less than " + (MIN_TIME_BEFORE_RELOGIN/1000) + " seconds"+ - " before."); + " before. Last Login=" + user.getLastLogin()); return false; } return true; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 602c694e874..d7b19afff8b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -445,6 +445,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final long DFS_BALANCER_GETBLOCKS_SIZE_DEFAULT = 2L*1024*1024*1024; // 2GB public static final String DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY = "dfs.balancer.getBlocks.min-block-size"; public static final long DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_DEFAULT = 10L*1024*1024; // 10MB + public static final String DFS_BALANCER_KEYTAB_ENABLED_KEY = "dfs.balancer.keytab.enabled"; + public static final boolean DFS_BALANCER_KEYTAB_ENABLED_DEFAULT = false; + public static final String DFS_BALANCER_ADDRESS_KEY = "dfs.balancer.address"; + public static final String DFS_BALANCER_ADDRESS_DEFAULT= "0.0.0.0:0"; + public static final String DFS_BALANCER_KEYTAB_FILE_KEY = "dfs.balancer.keytab.file"; + public static final String DFS_BALANCER_KERBEROS_PRINCIPAL_KEY = "dfs.balancer.kerberos.principal"; public static final String DFS_MOVER_MOVEDWINWIDTH_KEY = "dfs.mover.movedWinWidth"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java index 94d90dbdb05..fcd46695781 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java @@ -21,6 +21,7 @@ import static com.google.common.base.Preconditions.checkArgument; import java.io.IOException; import java.io.PrintStream; +import java.net.InetSocketAddress; import java.net.URI; import java.text.DateFormat; import java.util.Arrays; @@ -55,6 +56,9 @@ import org.apache.hadoop.hdfs.server.namenode.UnsupportedActionException; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport; import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.HostsFileReader; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; @@ -666,7 +670,7 @@ public class Balancer { LOG.info("included nodes = " + p.getIncludedNodes()); LOG.info("excluded nodes = " + p.getExcludedNodes()); LOG.info("source nodes = " + p.getSourceNodes()); - + checkKeytabAndInit(conf); System.out.println("Time Stamp Iteration# Bytes Already Moved Bytes Left To Move Bytes Being Moved"); List connectors = Collections.emptyList(); @@ -710,6 +714,22 @@ public class Balancer { return ExitStatus.SUCCESS.getExitCode(); } + private static void checkKeytabAndInit(Configuration conf) + throws IOException { + if (conf.getBoolean(DFSConfigKeys.DFS_BALANCER_KEYTAB_ENABLED_KEY, + DFSConfigKeys.DFS_BALANCER_KEYTAB_ENABLED_DEFAULT)) { + LOG.info("Keytab is configured, will login using keytab."); + UserGroupInformation.setConfiguration(conf); + String addr = conf.get(DFSConfigKeys.DFS_BALANCER_ADDRESS_KEY, + DFSConfigKeys.DFS_BALANCER_ADDRESS_DEFAULT); + InetSocketAddress socAddr = NetUtils.createSocketAddr(addr, 0, + DFSConfigKeys.DFS_BALANCER_ADDRESS_KEY); + SecurityUtil.login(conf, DFSConfigKeys.DFS_BALANCER_KEYTAB_FILE_KEY, + DFSConfigKeys.DFS_BALANCER_KERBEROS_PRINCIPAL_KEY, + socAddr.getHostName()); + } + } + /* Given elaspedTime in ms, return a printable string */ private static String time2Str(long elapsedTime) { String unit; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index a893addfc79..38bfedd3ac3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -1548,7 +1548,7 @@ The NameNode service principal. This is typically set to - nn/_HOST@REALM.TLD. Each NameNode will subsitute _HOST with its + nn/_HOST@REALM.TLD. Each NameNode will substitute _HOST with its own fully qualified hostname at startup. The _HOST placeholder allows using the same configuration setting on both NameNodes in an HA setup. @@ -1570,7 +1570,7 @@ The DataNode service principal. This is typically set to - dn/_HOST@REALM.TLD. Each DataNode will subsitute _HOST with its + dn/_HOST@REALM.TLD. Each DataNode will substitute _HOST with its own fully qualified hostname at startup. The _HOST placeholder allows using the same configuration setting on all DataNodes. @@ -1591,7 +1591,7 @@ The JournalNode service principal. This is typically set to - jn/_HOST@REALM.TLD. Each JournalNode will subsitute _HOST with its + jn/_HOST@REALM.TLD. Each JournalNode will substitute _HOST with its own fully qualified hostname at startup. The _HOST placeholder allows using the same configuration setting on all JournalNodes. @@ -4021,6 +4021,46 @@ + + dfs.balancer.keytab.enabled + false + + Set to true to enable login using a keytab for Kerberized Hadoop. + + + + + dfs.balancer.address + 0.0.0.0:0 + + The hostname used for a keytab based Kerberos login. Keytab based login + can be enabled with dfs.balancer.keytab.enabled. + + + + + dfs.balancer.keytab.file + + + The keytab file used by the Balancer to login as its + service principal. The principal name is configured with + dfs.balancer.kerberos.principal. Keytab based login can be + enabled with dfs.balancer.keytab.enabled. + + + + + dfs.balancer.kerberos.principal + + + The Balancer principal. This is typically set to + balancer/_HOST@REALM.TLD. The Balancer will substitute _HOST with its + own fully qualified hostname at startup. The _HOST placeholder + allows using the same configuration setting on different servers. + Keytab based login can be enabled with dfs.balancer.keytab.enabled. + + + ssl.server.truststore.location diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java index 59a19eaad19..3c67b20060d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java @@ -17,15 +17,32 @@ */ package org.apache.hadoop.hdfs.server.balancer; +import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SASL_KEY; import static org.apache.hadoop.fs.StorageType.DEFAULT; import static org.apache.hadoop.fs.StorageType.RAM_DISK; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DATA_TRANSFER_PROTECTION_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BALANCER_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BALANCER_KERBEROS_PRINCIPAL_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BALANCER_KEYTAB_ENABLED_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BALANCER_KEYTAB_FILE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_BLOCK_PINNING_ENABLED; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_LOCKED_MEMORY_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HTTP_POLICY_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -38,12 +55,14 @@ import java.io.PrintWriter; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.URI; +import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Properties; import java.util.Random; import java.util.Set; import java.util.concurrent.TimeoutException; @@ -55,6 +74,7 @@ import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.fs.permission.FsPermission; @@ -83,7 +103,12 @@ import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.LazyPersistTestCase; +import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.ssl.KeyStoreTestUtil; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Time; import org.apache.hadoop.util.Tool; @@ -107,8 +132,66 @@ public class TestBalancer { final static String RACK2 = "/rack2"; final private static String fileName = "/tmp.txt"; final static Path filePath = new Path(fileName); + final static private String username = "balancer"; + private static String principal; + private static File baseDir; + private static MiniKdc kdc; + private static File keytabFile; private MiniDFSCluster cluster; + static void initSecureConf(Configuration conf) throws Exception { + baseDir = new File(System.getProperty("test.build.dir", "target/test-dir"), + TestBalancer.class.getSimpleName()); + FileUtil.fullyDelete(baseDir); + assertTrue(baseDir.mkdirs()); + + Properties kdcConf = MiniKdc.createConf(); + kdc = new MiniKdc(kdcConf, baseDir); + kdc.start(); + + SecurityUtil.setAuthenticationMethod( + UserGroupInformation.AuthenticationMethod.KERBEROS, conf); + UserGroupInformation.setConfiguration(conf); + assertTrue("Expected configuration to enable security", + UserGroupInformation.isSecurityEnabled()); + + keytabFile = new File(baseDir, username + ".keytab"); + String keytab = keytabFile.getAbsolutePath(); + // Windows will not reverse name lookup "127.0.0.1" to "localhost". + String krbInstance = Path.WINDOWS ? "127.0.0.1" : "localhost"; + principal = username + "/" + krbInstance + "@" + kdc.getRealm(); + String spnegoPrincipal = "HTTP/" + krbInstance + "@" + kdc.getRealm(); + kdc.createPrincipal(keytabFile, username, username + "/" + krbInstance, + "HTTP/" + krbInstance); + + conf.set(DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY, principal); + conf.set(DFS_NAMENODE_KEYTAB_FILE_KEY, keytab); + conf.set(DFS_DATANODE_KERBEROS_PRINCIPAL_KEY, principal); + conf.set(DFS_DATANODE_KEYTAB_FILE_KEY, keytab); + conf.set(DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY, spnegoPrincipal); + conf.setBoolean(DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true); + conf.set(DFS_DATA_TRANSFER_PROTECTION_KEY, "authentication"); + conf.set(DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name()); + conf.set(DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0"); + conf.set(DFS_DATANODE_HTTPS_ADDRESS_KEY, "localhost:0"); + conf.setInt(IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SASL_KEY, 10); + + conf.setBoolean(DFS_BALANCER_KEYTAB_ENABLED_KEY, true); + conf.set(DFS_BALANCER_ADDRESS_KEY, "localhost:0"); + conf.set(DFS_BALANCER_KEYTAB_FILE_KEY, keytab); + conf.set(DFS_BALANCER_KERBEROS_PRINCIPAL_KEY, principal); + + String keystoresDir = baseDir.getAbsolutePath(); + String sslConfDir = KeyStoreTestUtil.getClasspathDir(TestBalancer.class); + KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); + + conf.set(DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getClientSSLConfigFileName()); + conf.set(DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getServerSSLConfigFileName()); + initConf(conf); + } + @After public void shutdown() throws Exception { if (cluster != null) { @@ -928,6 +1011,13 @@ public class TestBalancer { initConf(conf); oneNodeTest(conf, false); } + + @Test(timeout = 100000) + public void testUnknownDatanodeSimple() throws Exception { + Configuration conf = new HdfsConfiguration(); + initConf(conf); + testUnknownDatanode(conf); + } /* we first start a cluster and fill the cluster up to a certain size. * then redistribute blocks according the required distribution. @@ -936,10 +1026,8 @@ public class TestBalancer { * A partially filled datanode is excluded during balancing. * This triggers a situation where one of the block's location is unknown. */ - @Test(timeout=100000) - public void testUnknownDatanode() throws Exception { - Configuration conf = new HdfsConfiguration(); - initConf(conf); + private void testUnknownDatanode(Configuration conf) + throws IOException, InterruptedException, TimeoutException { long distribution[] = new long[] {50*CAPACITY/100, 70*CAPACITY/100, 0*CAPACITY/100}; long capacities[] = new long[]{CAPACITY, CAPACITY, CAPACITY}; String racks[] = new String[] {RACK0, RACK1, RACK1}; @@ -1759,6 +1847,33 @@ public class TestBalancer { } } + /** + * Test Balancer runs fine when logging in with a keytab in kerberized env. + * Reusing testUnknownDatanode here for basic functionality testing. + */ + @Test(timeout = 300000) + public void testBalancerWithKeytabs() throws Exception { + final Configuration conf = new HdfsConfiguration(); + initSecureConf(conf); + final UserGroupInformation ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI( + principal, keytabFile.getAbsolutePath()); + try { + ugi.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + // verify that balancer runs Ok. + testUnknownDatanode(conf); + // verify that UGI was logged in using keytab. + assertTrue(UserGroupInformation.isLoginKeytabBased()); + return null; + } + }); + } finally { + // Reset UGI so that other tests are not affected. + UserGroupInformation.setConfiguration(new Configuration()); + } + } + /** * @param args */