From 9b08654536889650c611f4a2f794822bb192650b Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Tue, 3 Dec 2013 03:31:21 +0000 Subject: [PATCH 01/32] HDFS-5560. Trash configuration log statements prints incorrect units. Contributed by Josh Elser. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1547266 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../main/java/org/apache/hadoop/fs/TrashPolicyDefault.java | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 8a0af475756..61d29f43aeb 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -519,6 +519,9 @@ Release 2.3.0 - UNRELEASED HADOOP-10130. RawLocalFS::LocalFSFileInputStream.pread does not track FS::Statistics (Binglin Chang via Colin Patrick McCabe) + HDFS-5560. Trash configuration log statements prints incorrect units. + (Josh Elser via Andrew Wang) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java index a8f49c243f4..cfb51e24ec7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java @@ -90,8 +90,8 @@ public class TrashPolicyDefault extends TrashPolicy { FS_TRASH_CHECKPOINT_INTERVAL_KEY, FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT) * MSECS_PER_MINUTE); LOG.info("Namenode trash configuration: Deletion interval = " + - this.deletionInterval + " minutes, Emptier interval = " + - this.emptierInterval + " minutes."); + (this.deletionInterval / MSECS_PER_MINUTE) + " minutes, Emptier interval = " + + (this.emptierInterval / MSECS_PER_MINUTE) + " minutes."); } private Path makeTrashRelativePath(Path basePath, Path rmFilePath) { From 7af12ab207cf6e5e63ccb359e928db7fe2350585 Mon Sep 17 00:00:00 2001 From: Kihwal Lee Date: Tue, 3 Dec 2013 14:12:43 +0000 Subject: [PATCH 02/32] HDFS-5558. LeaseManager monitor thread can crash if the last block is complete but another block is not. Contributed by Kihwal Lee. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1547393 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 8 +++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 62eb868820a..1177771ec98 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -4063,6 +4063,9 @@ Release 0.23.10 - UNRELEASED HDFS-5557. Write pipeline recovery for the last packet in the block may cause rejection of valid replicas. (kihwal) + HDFS-5558. LeaseManager monitor thread can crash if the last block is + complete but another block is not. (kihwal) + Release 0.23.9 - 2013-07-08 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 6f9e1a48a1b..0cae09863b8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -2933,6 +2933,12 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } throw lee; } + // Check the state of the penultimate block. It should be completed + // before attempting to complete the last one. + if (!checkFileProgress(pendingFile, false)) { + return false; + } + // commit the last block and complete it if it has minimum replicas commitOrCompleteLastBlock(pendingFile, last); @@ -3002,7 +3008,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, // BlockInfo b = v.getPenultimateBlock(); if (b != null && !b.isComplete()) { - LOG.info("BLOCK* checkFileProgress: " + b + LOG.warn("BLOCK* checkFileProgress: " + b + " has not reached minimal replication " + blockManager.minReplication); return false; From 9c95015bb47f362e592393f0cec899fd8fdd3b04 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Tue, 3 Dec 2013 19:28:02 +0000 Subject: [PATCH 03/32] YARN-895. Changed RM state-store to not crash immediately if RM restarts while the state-store is down. Contributed by Jian He. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1547538 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../dev-support/findbugs-exclude.xml | 5 ++ .../hadoop/yarn/conf/YarnConfiguration.java | 18 +++++-- .../src/main/resources/yarn-default.xml | 41 ++++++++++++---- .../recovery/FileSystemRMStateStore.java | 9 +++- .../recovery/ZKRMStateStore.java | 7 +++ .../recovery/TestFSRMStateStore.java | 49 +++++++++++++++++++ ...TestZKRMStateStoreZKClientConnections.java | 32 ++++++++++++ 8 files changed, 148 insertions(+), 16 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 5cd354cfac9..adbcdd00500 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -191,6 +191,9 @@ Release 2.4.0 - UNRELEASED YARN-1416. Fixed a few invalid transitions in RMApp, RMAppAttempt and in some tests. (Jian He via vinodkv) + YARN-895. Changed RM state-store to not crash immediately if RM restarts while + the state-store is down. (Jian He via vinodkv) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index b25466533a3..80598a43bb8 100644 --- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -305,4 +305,9 @@ + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index af48f26298a..0480cab7d2d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -301,22 +301,30 @@ public class YarnConfiguration extends Configuration { public static final String RM_STORE = RM_PREFIX + "store.class"; /** URI for FileSystemRMStateStore */ - public static final String FS_RM_STATE_STORE_URI = - RM_PREFIX + "fs.state-store.uri"; + public static final String FS_RM_STATE_STORE_URI = RM_PREFIX + + "fs.state-store.uri"; + public static final String FS_RM_STATE_STORE_RETRY_POLICY_SPEC = RM_PREFIX + + "fs.state-store.retry-policy-spec"; + public static final String DEFAULT_FS_RM_STATE_STORE_RETRY_POLICY_SPEC = + "2000, 500"; /** * Comma separated host:port pairs, each corresponding to a ZK server for * ZKRMStateStore */ public static final String ZK_STATE_STORE_PREFIX = - RM_PREFIX + "zk.state-store."; + RM_PREFIX + "zk-state-store."; public static final String ZK_RM_STATE_STORE_NUM_RETRIES = ZK_STATE_STORE_PREFIX + "num-retries"; - public static final int DEFAULT_ZK_RM_STATE_STORE_NUM_RETRIES = 3; + public static final int DEFAULT_ZK_RM_STATE_STORE_NUM_RETRIES = 500; + /** retry interval when connecting to zookeeper*/ + public static final String ZK_RM_STATE_STORE_RETRY_INTERVAL_MS = + ZK_STATE_STORE_PREFIX + "retry-interval-ms"; + public static final long DEFAULT_ZK_RM_STATE_STORE_RETRY_INTERVAL_MS = 2000; public static final String ZK_RM_STATE_STORE_ADDRESS = ZK_STATE_STORE_PREFIX + "address"; /** Timeout in millisec for ZK server connection for ZKRMStateStore */ public static final String ZK_RM_STATE_STORE_TIMEOUT_MS = - ZK_STATE_STORE_PREFIX + "timeout.ms"; + ZK_STATE_STORE_PREFIX + "timeout-ms"; public static final int DEFAULT_ZK_RM_STATE_STORE_TIMEOUT_MS = 60000; /** Parent znode path under which ZKRMStateStore will create znodes */ public static final String ZK_RM_STATE_STORE_PARENT_PATH = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 7f6e050d2b5..d798f4c9767 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -283,8 +283,8 @@ is implicitly fenced, meaning a single ResourceManager is able to use the store at any point in time. More details on this, along with setting up appropriate ACLs is discussed under the description for - yarn.resourcemanager.zk.state-store.root-node.acl. - yarn.resourcemanager.zk.state-store.address + yarn.resourcemanager.zk-state-store.root-node.acl. + yarn.resourcemanager.zk-state-store.address @@ -293,8 +293,15 @@ ZooKeeper. This may be supplied when using org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore as the value for yarn.resourcemanager.store.class - yarn.resourcemanager.zk.state-store.num-retries - 3 + yarn.resourcemanager.zk-state-store.num-retries + 500 + + + + Retry interval in milliseconds when ZKRMStateStore tries to + connect to ZooKeeper. + yarn.resourcemanager.zk-state-store.retry-interval-ms + 2000 @@ -302,16 +309,20 @@ stored. This must be supplied when using org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore as the value for yarn.resourcemanager.store.class - yarn.resourcemanager.zk.state-store.parent-path + yarn.resourcemanager.zk-state-store.parent-path /rmstore - Timeout when connecting to ZooKeeper. + ZooKeeper session timeout in milliseconds. Session expiration + is managed by the ZooKeeper cluster itself, not by the client. This value is + used by the cluster to determine when the client's session expires. + Expirations happens when the cluster does not hear from the client within + the specified session timeout period (i.e. no heartbeat). This may be supplied when using org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore as the value for yarn.resourcemanager.store.class - yarn.resourcemanager.zk.state-store.timeout.ms + yarn.resourcemanager.zk-state-store.timeout-ms 60000 @@ -320,7 +331,7 @@ This may be supplied when using org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore as the value for yarn.resourcemanager.store.class - yarn.resourcemanager.zk.state-store.acl + yarn.resourcemanager.zk-state-store.acl world:anyone:rwcda @@ -336,7 +347,7 @@ permissions. By default, when this property is not set, we use the ACLs from - yarn.resourcemanager.zk.state-store.acl for shared admin access and + yarn.resourcemanager.zk-state-store.acl for shared admin access and rm-address:cluster-timestamp for username-based exclusive create-delete access. @@ -346,7 +357,7 @@ ResourceManagers have shared admin access and the Active ResourceManger takes over (exclusively) the create-delete access. - yarn.resourcemanager.zk.state-store.root-node.acl + yarn.resourcemanager.zk-state-store.root-node.acl @@ -359,6 +370,16 @@ + + hdfs client retry policy specification. hdfs client retry + is always enabled. Specified in pairs of sleep-time and number-of-retries + and (t0, n0), (t1, n1), ..., the first n0 retries sleep t0 milliseconds on + average, the following n1 retries sleep t1 milliseconds on average, and so on. + + yarn.resourcemanager.fs.state-store.retry-policy-spec + 2000, 500 + + Enable RM high-availability. When enabled, (1) The RM starts in the Standby mode by default, and transitions to diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java index 2ef6bcd62c0..23cefd334db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java @@ -94,7 +94,14 @@ public class FileSystemRMStateStore extends RMStateStore { // create filesystem only now, as part of service-start. By this time, RM is // authenticated with kerberos so we are good to create a file-system // handle. - fs = fsWorkingPath.getFileSystem(getConfig()); + Configuration conf = new Configuration(getConfig()); + conf.setBoolean("dfs.client.retry.policy.enabled", true); + String retryPolicy = + conf.get(YarnConfiguration.FS_RM_STATE_STORE_RETRY_POLICY_SPEC, + YarnConfiguration.DEFAULT_FS_RM_STATE_STORE_RETRY_POLICY_SPEC); + conf.set("dfs.client.retry.policy.spec", retryPolicy); + + fs = fsWorkingPath.getFileSystem(conf); fs.mkdirs(rmDTSecretManagerRoot); fs.mkdirs(rmAppRoot); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java index 1621d8327a5..f419ff09888 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java @@ -82,6 +82,7 @@ public class ZKRMStateStore extends RMStateStore { private String zkHostPort = null; private int zkSessionTimeout; + private long zkRetryInterval; private List zkAcl; private String zkRootNodePath; private String rmDTSecretManagerRoot; @@ -161,6 +162,9 @@ public class ZKRMStateStore extends RMStateStore { zkSessionTimeout = conf.getInt(YarnConfiguration.ZK_RM_STATE_STORE_TIMEOUT_MS, YarnConfiguration.DEFAULT_ZK_RM_STATE_STORE_TIMEOUT_MS); + zkRetryInterval = + conf.getLong(YarnConfiguration.ZK_RM_STATE_STORE_RETRY_INTERVAL_MS, + YarnConfiguration.DEFAULT_ZK_RM_STATE_STORE_RETRY_INTERVAL_MS); // Parse authentication from configuration. String zkAclConf = conf.get(YarnConfiguration.ZK_RM_STATE_STORE_ACL, @@ -810,6 +814,9 @@ public class ZKRMStateStore extends RMStateStore { } } catch (KeeperException ke) { if (shouldRetry(ke.code()) && ++retry < numRetries) { + LOG.info("Waiting for zookeeper to be connected, retry no. + " + + retry); + Thread.sleep(zkRetryInterval); continue; } throw ke; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java index 63fe97557c5..4df1c3b0c83 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java @@ -19,6 +19,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.recovery; import static org.junit.Assert.assertTrue; + +import java.util.concurrent.atomic.AtomicBoolean; + import junit.framework.Assert; import org.apache.commons.logging.Log; @@ -33,7 +36,9 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.RMStateVersionPBImpl; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.util.ConverterUtils; import org.junit.Test; @@ -81,6 +86,8 @@ public class TestFSRMStateStore extends RMStateStoreTestBase { YarnConfiguration conf = new YarnConfiguration(); conf.set(YarnConfiguration.FS_RM_STATE_STORE_URI, workingDirPathURI.toString()); + conf.set(YarnConfiguration.FS_RM_STATE_STORE_RETRY_POLICY_SPEC, + "100,6000"); this.store = new TestFileSystemRMStore(conf); return store; } @@ -139,4 +146,46 @@ public class TestFSRMStateStore extends RMStateStoreTestBase { cluster.shutdown(); } } + + @Test (timeout = 30000) + public void testFSRMStateStoreClientRetry() throws Exception { + HdfsConfiguration conf = new HdfsConfiguration(); + MiniDFSCluster cluster = + new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + cluster.waitActive(); + try { + TestFSRMStateStoreTester fsTester = new TestFSRMStateStoreTester(cluster); + final RMStateStore store = fsTester.getRMStateStore(); + store.setRMDispatcher(new TestDispatcher()); + final AtomicBoolean assertionFailedInThread = new AtomicBoolean(false); + cluster.shutdownNameNodes(); + + Thread clientThread = new Thread() { + @Override + public void run() { + try { + store.storeApplicationStateInternal("application1", + (ApplicationStateDataPBImpl) ApplicationStateDataPBImpl + .newApplicationStateData(111, 111, "user", null, + RMAppState.ACCEPTED, "diagnostics", 333)); + } catch (Exception e) { + // TODO 0 datanode exception will not be retried by dfs client, fix + // that separately. + if (!e.getMessage().contains("could only be replicated" + + " to 0 nodes instead of minReplication (=1)")) { + assertionFailedInThread.set(true); + } + e.printStackTrace(); + } + } + }; + Thread.sleep(2000); + clientThread.start(); + cluster.restartNameNode(); + clientThread.join(); + Assert.assertFalse(assertionFailedInThread.get()); + } finally { + cluster.shutdown(); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java index 82e550c9173..3def83c0ad7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java @@ -37,6 +37,7 @@ import org.junit.Test; import java.io.IOException; import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.atomic.AtomicBoolean; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -114,6 +115,37 @@ public class TestZKRMStateStoreZKClientConnections extends } } + @Test (timeout = 20000) + public void testZKClientRetry() throws Exception { + TestZKClient zkClientTester = new TestZKClient(); + final String path = "/test"; + YarnConfiguration conf = new YarnConfiguration(); + conf.setInt(YarnConfiguration.ZK_RM_STATE_STORE_TIMEOUT_MS, 100); + conf.setLong(YarnConfiguration.ZK_RM_STATE_STORE_RETRY_INTERVAL_MS, 100); + final ZKRMStateStore store = + (ZKRMStateStore) zkClientTester.getRMStateStore(conf); + TestDispatcher dispatcher = new TestDispatcher(); + store.setRMDispatcher(dispatcher); + final AtomicBoolean assertionFailedInThread = new AtomicBoolean(false); + + stopServer(); + Thread clientThread = new Thread() { + @Override + public void run() { + try { + store.getDataWithRetries(path, true); + } catch (Exception e) { + e.printStackTrace(); + assertionFailedInThread.set(true); + } + } + }; + Thread.sleep(2000); + startServer(); + clientThread.join(); + Assert.assertFalse(assertionFailedInThread.get()); + } + @Test(timeout = 20000) public void testZKClientDisconnectAndReconnect() throws Exception { From 5afc1242ea323d5d160b8a1c676e499af81f21b4 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Eagles Date: Tue, 3 Dec 2013 22:44:07 +0000 Subject: [PATCH 04/32] MAPREDUCE-5645. TestFixedLengthInputFormat fails with native libs (Mit Desai via jeagles) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1547624 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../mapred/TestFixedLengthInputFormat.java | 22 +++++++++++-------- .../lib/input/TestFixedLengthInputFormat.java | 18 +++++++++------ 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 1c2509584a1..a9527bc4bac 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -226,6 +226,9 @@ Release 2.4.0 - UNRELEASED MAPREDUCE-5631. TestJobEndNotifier.testNotifyRetries fails with Should have taken more than 5 seconds in jdk7 (Jonathan Eagles via jlowe) + MAPREDUCE-5645. TestFixedLengthInputFormat fails with native libs (Mit + Desai via jeagles) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFixedLengthInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFixedLengthInputFormat.java index a5f19eeb25d..d0bd7eecdf3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFixedLengthInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFixedLengthInputFormat.java @@ -197,17 +197,17 @@ public class TestFixedLengthInputFormat { public void testGzipWithTwoInputs() throws IOException { CompressionCodec gzip = new GzipCodec(); localFs.delete(workDir, true); - // Create files with fixed length records with 5 byte long records. - writeFile(localFs, new Path(workDir, "part1.txt.gz"), gzip, - "one two threefour five six seveneightnine ten "); - writeFile(localFs, new Path(workDir, "part2.txt.gz"), gzip, - "ten nine eightsevensix five four threetwo one "); FixedLengthInputFormat format = new FixedLengthInputFormat(); JobConf job = new JobConf(defaultConf); format.setRecordLength(job, 5); FileInputFormat.setInputPaths(job, workDir); ReflectionUtils.setConf(gzip, job); format.configure(job); + // Create files with fixed length records with 5 byte long records. + writeFile(localFs, new Path(workDir, "part1.txt.gz"), gzip, + "one two threefour five six seveneightnine ten "); + writeFile(localFs, new Path(workDir, "part2.txt.gz"), gzip, + "ten nine eightsevensix five four threetwo one "); InputSplit[] splits = format.getSplits(job, 100); assertEquals("compressed splits == 2", 2, splits.length); FileSplit tmp = (FileSplit) splits[0]; @@ -283,12 +283,16 @@ public class TestFixedLengthInputFormat { int fileSize = (totalRecords * recordLength); LOG.info("totalRecords=" + totalRecords + " recordLength=" + recordLength); + // Create the job + JobConf job = new JobConf(defaultConf); + if (codec != null) { + ReflectionUtils.setConf(codec, job); + } // Create the test file ArrayList recordList = createFile(file, codec, recordLength, totalRecords); assertTrue(localFs.exists(file)); - // Create the job and set the fixed length record length config property - JobConf job = new JobConf(defaultConf); + //set the fixed length record length config property for the job FixedLengthInputFormat.setRecordLength(job, recordLength); int numSplits = 1; @@ -383,8 +387,6 @@ public class TestFixedLengthInputFormat { if (codec != null) { fileName.append(".gz"); } - writeFile(localFs, new Path(workDir, fileName.toString()), codec, - "one two threefour five six seveneightnine ten"); FixedLengthInputFormat format = new FixedLengthInputFormat(); JobConf job = new JobConf(defaultConf); format.setRecordLength(job, 5); @@ -393,6 +395,8 @@ public class TestFixedLengthInputFormat { ReflectionUtils.setConf(codec, job); } format.configure(job); + writeFile(localFs, new Path(workDir, fileName.toString()), codec, + "one two threefour five six seveneightnine ten"); InputSplit[] splits = format.getSplits(job, 100); if (codec != null) { assertEquals("compressed splits == 1", 1, splits.length); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFixedLengthInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFixedLengthInputFormat.java index f00b1a92ac1..cf7c61c09b5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFixedLengthInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFixedLengthInputFormat.java @@ -225,16 +225,16 @@ public class TestFixedLengthInputFormat { public void testGzipWithTwoInputs() throws Exception { CompressionCodec gzip = new GzipCodec(); localFs.delete(workDir, true); - // Create files with fixed length records with 5 byte long records. - writeFile(localFs, new Path(workDir, "part1.txt.gz"), gzip, - "one two threefour five six seveneightnine ten "); - writeFile(localFs, new Path(workDir, "part2.txt.gz"), gzip, - "ten nine eightsevensix five four threetwo one "); Job job = Job.getInstance(defaultConf); FixedLengthInputFormat format = new FixedLengthInputFormat(); format.setRecordLength(job.getConfiguration(), 5); ReflectionUtils.setConf(gzip, job.getConfiguration()); FileInputFormat.setInputPaths(job, workDir); + // Create files with fixed length records with 5 byte long records. + writeFile(localFs, new Path(workDir, "part1.txt.gz"), gzip, + "one two threefour five six seveneightnine ten "); + writeFile(localFs, new Path(workDir, "part2.txt.gz"), gzip, + "ten nine eightsevensix five four threetwo one "); List splits = format.getSplits(job); assertEquals("compressed splits == 2", 2, splits.size()); FileSplit tmp = (FileSplit) splits.get(0); @@ -310,12 +310,16 @@ public class TestFixedLengthInputFormat { int fileSize = (totalRecords * recordLength); LOG.info("totalRecords=" + totalRecords + " recordLength=" + recordLength); + // Create the job + Job job = Job.getInstance(defaultConf); + if (codec != null) { + ReflectionUtils.setConf(codec, job.getConfiguration()); + } // Create the test file ArrayList recordList = createFile(file, codec, recordLength, totalRecords); assertTrue(localFs.exists(file)); - // Create the job and set the fixed length record length config property - Job job = Job.getInstance(defaultConf); + //set the fixed length record length config property for the job FixedLengthInputFormat.setRecordLength(job.getConfiguration(), recordLength); From f7fe50d55f51257c7b38724b37b9976a96f98c38 Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Tue, 3 Dec 2013 22:44:46 +0000 Subject: [PATCH 05/32] HADOOP-10127. Add ipc.client.connect.retry.interval to control the frequency of connection retries (Karthik Kambatla via Sandy Ryza) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1547626 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../apache/hadoop/fs/CommonConfigurationKeysPublic.java | 5 +++++ .../src/main/java/org/apache/hadoop/ipc/Client.java | 7 ++++++- .../hadoop-common/src/main/resources/core-default.xml | 8 ++++++++ 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 61d29f43aeb..23e82b79462 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -390,6 +390,9 @@ Release 2.4.0 - UNRELEASED HADOOP-10126. LightWeightGSet log message is confusing. (Vinay via suresh) + HADOOP-10127. Add ipc.client.connect.retry.interval to control the frequency + of connection retries (Karthik Kambatla via Sandy Ryza) + OPTIMIZATIONS HADOOP-9748. Reduce blocking on UGI.ensureInitialized (daryn) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 5a75cbdf15b..87746bbe396 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -192,6 +192,11 @@ public class CommonConfigurationKeysPublic { /** Default value for IPC_CLIENT_CONNECT_MAX_RETRIES_KEY */ public static final int IPC_CLIENT_CONNECT_MAX_RETRIES_DEFAULT = 10; /** See core-default.xml */ + public static final String IPC_CLIENT_CONNECT_RETRY_INTERVAL_KEY = + "ipc.client.connect.retry.interval"; + /** Default value for IPC_CLIENT_CONNECT_RETRY_INTERVAL_KEY */ + public static final int IPC_CLIENT_CONNECT_RETRY_INTERVAL_DEFAULT = 1000; + /** See core-default.xml */ public static final String IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY = "ipc.client.connect.max.retries.on.timeouts"; /** Default value for IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY */ diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java index b444a115cb5..764c7dca5ca 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java @@ -1562,8 +1562,13 @@ public class Client { final int max = conf.getInt( CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_DEFAULT); + final int retryInterval = conf.getInt( + CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_RETRY_INTERVAL_KEY, + CommonConfigurationKeysPublic + .IPC_CLIENT_CONNECT_RETRY_INTERVAL_DEFAULT); + connectionRetryPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep( - max, 1, TimeUnit.SECONDS); + max, retryInterval, TimeUnit.MILLISECONDS); } boolean doPing = diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 746d209715a..c7671c5451f 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -618,6 +618,14 @@ + + ipc.client.connect.retry.interval + 1000 + Indicates the number of milliseconds a client will wait for + before retrying to establish a server connection. + + + ipc.client.connect.timeout 20000 From 61c45aae4d0a6cc82b0e9253753ec73cbb0abb7b Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Tue, 3 Dec 2013 23:13:02 +0000 Subject: [PATCH 06/32] HDFS-4997. libhdfs does not return correct error code in most cases (cmccabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1547637 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../contrib/libwebhdfs/src/hdfs_json_parser.c | 9 -- .../src/main/native/libhdfs/exception.c | 33 ++++- .../native/libhdfs/test_libhdfs_threaded.c | 123 +++++++++++------- 4 files changed, 106 insertions(+), 61 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 1177771ec98..4ac7567a988 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -767,6 +767,8 @@ Release 2.3.0 - UNRELEASED HDFS-5563. NFS gateway should commit the buffered data when read request comes after write to the same file (brandonli) + HDFS-4997. libhdfs doesn't return correct error codes in most cases (cmccabe) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_json_parser.c b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_json_parser.c index a883f06f7c6..178fb9de664 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_json_parser.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_json_parser.c @@ -39,14 +39,6 @@ struct jsonException { const char *message; }; -static void dotsToSlashes(char *str) -{ - for (; *str != '\0'; str++) { - if (*str == '.') - *str = '/'; - } -} - /** Print out the JSON exception information */ static int printJsonExceptionV(struct jsonException *exc, int noPrintFlags, const char *fmt, va_list ap) @@ -62,7 +54,6 @@ static int printJsonExceptionV(struct jsonException *exc, int noPrintFlags, fprintf(stderr, "printJsonExceptionV: internal out of memory error\n"); return EINTERNAL; } - dotsToSlashes(javaClassName); getExceptionInfo(javaClassName, noPrintFlags, &excErrno, &shouldPrint); free(javaClassName); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/exception.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/exception.c index 6a50c981b96..d7e17208454 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/exception.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/exception.c @@ -35,36 +35,55 @@ struct ExceptionInfo { static const struct ExceptionInfo gExceptionInfo[] = { { - .name = "java/io/FileNotFoundException", + .name = "java.io.FileNotFoundException", .noPrintFlag = NOPRINT_EXC_FILE_NOT_FOUND, .excErrno = ENOENT, }, { - .name = "org/apache/hadoop/security/AccessControlException", + .name = "org.apache.hadoop.security.AccessControlException", .noPrintFlag = NOPRINT_EXC_ACCESS_CONTROL, .excErrno = EACCES, }, { - .name = "org/apache/hadoop/fs/UnresolvedLinkException", + .name = "org.apache.hadoop.fs.UnresolvedLinkException", .noPrintFlag = NOPRINT_EXC_UNRESOLVED_LINK, .excErrno = ENOLINK, }, { - .name = "org/apache/hadoop/fs/ParentNotDirectoryException", + .name = "org.apache.hadoop.fs.ParentNotDirectoryException", .noPrintFlag = NOPRINT_EXC_PARENT_NOT_DIRECTORY, .excErrno = ENOTDIR, }, { - .name = "java/lang/IllegalArgumentException", + .name = "java.lang.IllegalArgumentException", .noPrintFlag = NOPRINT_EXC_ILLEGAL_ARGUMENT, .excErrno = EINVAL, }, { - .name = "java/lang/OutOfMemoryError", + .name = "java.lang.OutOfMemoryError", .noPrintFlag = 0, .excErrno = ENOMEM, }, - + { + .name = "org.apache.hadoop.hdfs.server.namenode.SafeModeException", + .noPrintFlag = 0, + .excErrno = EROFS, + }, + { + .name = "org.apache.hadoop.fs.FileAlreadyExistsException", + .noPrintFlag = 0, + .excErrno = EEXIST, + }, + { + .name = "org.apache.hadoop.hdfs.protocol.QuotaExceededException", + .noPrintFlag = 0, + .excErrno = EDQUOT, + }, + { + .name = "org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException", + .noPrintFlag = 0, + .excErrno = ESTALE, + }, }; void getExceptionInfo(const char *excName, int noPrintFlags, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c index cf2bf963922..3a8f31dccac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c @@ -48,7 +48,8 @@ struct tlhThreadInfo { pthread_t thread; }; -static int hdfsSingleNameNodeConnect(struct NativeMiniDfsCluster *cl, hdfsFS *fs) +static int hdfsSingleNameNodeConnect(struct NativeMiniDfsCluster *cl, hdfsFS *fs, + const char *username) { int ret, port; hdfsFS hdfs; @@ -70,6 +71,9 @@ static int hdfsSingleNameNodeConnect(struct NativeMiniDfsCluster *cl, hdfsFS *fs TO_STR(TLH_DEFAULT_BLOCK_SIZE)); hdfsBuilderConfSetStr(bld, "dfs.blocksize", TO_STR(TLH_DEFAULT_BLOCK_SIZE)); + if (username) { + hdfsBuilderSetUserName(bld, username); + } hdfs = hdfsBuilderConnect(bld); if (!hdfs) { ret = -errno; @@ -110,36 +114,58 @@ static int doTestGetDefaultBlockSize(hdfsFS fs, const char *path) return 0; } -static int doTestHdfsOperations(struct tlhThreadInfo *ti, hdfsFS fs) +struct tlhPaths { + char prefix[256]; + char file1[256]; + char file2[256]; +}; + +static int setupPaths(const struct tlhThreadInfo *ti, struct tlhPaths *paths) { - char prefix[256], tmp[256]; + memset(paths, sizeof(*paths), 0); + if (snprintf(paths->prefix, sizeof(paths->prefix), "/tlhData%04d", + ti->threadIdx) >= sizeof(paths->prefix)) { + return ENAMETOOLONG; + } + if (snprintf(paths->file1, sizeof(paths->file1), "%s/file1", + paths->prefix) >= sizeof(paths->file1)) { + return ENAMETOOLONG; + } + if (snprintf(paths->file2, sizeof(paths->file2), "%s/file2", + paths->prefix) >= sizeof(paths->file2)) { + return ENAMETOOLONG; + } + return 0; +} + +static int doTestHdfsOperations(struct tlhThreadInfo *ti, hdfsFS fs, + const struct tlhPaths *paths) +{ + char tmp[4096]; hdfsFile file; int ret, expected; hdfsFileInfo *fileInfo; struct hdfsReadStatistics *readStats = NULL; - snprintf(prefix, sizeof(prefix), "/tlhData%04d", ti->threadIdx); - - if (hdfsExists(fs, prefix) == 0) { - EXPECT_ZERO(hdfsDelete(fs, prefix, 1)); + if (hdfsExists(fs, paths->prefix) == 0) { + EXPECT_ZERO(hdfsDelete(fs, paths->prefix, 1)); } - EXPECT_ZERO(hdfsCreateDirectory(fs, prefix)); - snprintf(tmp, sizeof(tmp), "%s/file", prefix); + EXPECT_ZERO(hdfsCreateDirectory(fs, paths->prefix)); - EXPECT_ZERO(doTestGetDefaultBlockSize(fs, prefix)); + EXPECT_ZERO(doTestGetDefaultBlockSize(fs, paths->prefix)); /* There should not be any file to open for reading. */ - EXPECT_NULL(hdfsOpenFile(fs, tmp, O_RDONLY, 0, 0, 0)); + EXPECT_NULL(hdfsOpenFile(fs, paths->file1, O_RDONLY, 0, 0, 0)); /* hdfsOpenFile should not accept mode = 3 */ - EXPECT_NULL(hdfsOpenFile(fs, tmp, 3, 0, 0, 0)); + EXPECT_NULL(hdfsOpenFile(fs, paths->file1, 3, 0, 0, 0)); - file = hdfsOpenFile(fs, tmp, O_WRONLY, 0, 0, 0); + file = hdfsOpenFile(fs, paths->file1, O_WRONLY, 0, 0, 0); EXPECT_NONNULL(file); /* TODO: implement writeFully and use it here */ - expected = strlen(prefix); - ret = hdfsWrite(fs, file, prefix, expected); + expected = strlen(paths->prefix); + ret = hdfsWrite(fs, file, paths->prefix, expected); if (ret < 0) { ret = errno; fprintf(stderr, "hdfsWrite failed and set errno %d\n", ret); @@ -155,7 +181,7 @@ static int doTestHdfsOperations(struct tlhThreadInfo *ti, hdfsFS fs) EXPECT_ZERO(hdfsCloseFile(fs, file)); /* Let's re-open the file for reading */ - file = hdfsOpenFile(fs, tmp, O_RDONLY, 0, 0, 0); + file = hdfsOpenFile(fs, paths->file1, O_RDONLY, 0, 0, 0); EXPECT_NONNULL(file); EXPECT_ZERO(hdfsFileGetReadStatistics(file, &readStats)); @@ -180,60 +206,67 @@ static int doTestHdfsOperations(struct tlhThreadInfo *ti, hdfsFS fs) errno = 0; EXPECT_INT_EQ(expected, readStats->totalBytesRead); hdfsFileFreeReadStatistics(readStats); - EXPECT_ZERO(memcmp(prefix, tmp, expected)); + EXPECT_ZERO(memcmp(paths->prefix, tmp, expected)); EXPECT_ZERO(hdfsCloseFile(fs, file)); // TODO: Non-recursive delete should fail? //EXPECT_NONZERO(hdfsDelete(fs, prefix, 0)); + EXPECT_ZERO(hdfsCopy(fs, paths->file1, fs, paths->file2)); - snprintf(tmp, sizeof(tmp), "%s/file", prefix); - EXPECT_ZERO(hdfsChown(fs, tmp, NULL, NULL)); - EXPECT_ZERO(hdfsChown(fs, tmp, NULL, "doop")); - fileInfo = hdfsGetPathInfo(fs, tmp); + EXPECT_ZERO(hdfsChown(fs, paths->file2, NULL, NULL)); + EXPECT_ZERO(hdfsChown(fs, paths->file2, NULL, "doop")); + fileInfo = hdfsGetPathInfo(fs, paths->file2); EXPECT_NONNULL(fileInfo); EXPECT_ZERO(strcmp("doop", fileInfo->mGroup)); hdfsFreeFileInfo(fileInfo, 1); - EXPECT_ZERO(hdfsChown(fs, tmp, "ha", "doop2")); - fileInfo = hdfsGetPathInfo(fs, tmp); + EXPECT_ZERO(hdfsChown(fs, paths->file2, "ha", "doop2")); + fileInfo = hdfsGetPathInfo(fs, paths->file2); EXPECT_NONNULL(fileInfo); EXPECT_ZERO(strcmp("ha", fileInfo->mOwner)); EXPECT_ZERO(strcmp("doop2", fileInfo->mGroup)); hdfsFreeFileInfo(fileInfo, 1); - EXPECT_ZERO(hdfsChown(fs, tmp, "ha2", NULL)); - fileInfo = hdfsGetPathInfo(fs, tmp); + EXPECT_ZERO(hdfsChown(fs, paths->file2, "ha2", NULL)); + fileInfo = hdfsGetPathInfo(fs, paths->file2); EXPECT_NONNULL(fileInfo); EXPECT_ZERO(strcmp("ha2", fileInfo->mOwner)); EXPECT_ZERO(strcmp("doop2", fileInfo->mGroup)); hdfsFreeFileInfo(fileInfo, 1); - EXPECT_ZERO(hdfsDelete(fs, prefix, 1)); + snprintf(tmp, sizeof(tmp), "%s/nonexistent-file-name", paths->prefix); + EXPECT_NEGATIVE_ONE_WITH_ERRNO(hdfsChown(fs, tmp, "ha3", NULL), ENOENT); + return 0; +} + +static int testHdfsOperationsImpl(struct tlhThreadInfo *ti) +{ + hdfsFS fs = NULL; + struct tlhPaths paths; + + fprintf(stderr, "testHdfsOperations(threadIdx=%d): starting\n", + ti->threadIdx); + EXPECT_ZERO(hdfsSingleNameNodeConnect(tlhCluster, &fs, NULL)); + EXPECT_ZERO(setupPaths(ti, &paths)); + // test some operations + EXPECT_ZERO(doTestHdfsOperations(ti, fs, &paths)); + EXPECT_ZERO(hdfsDisconnect(fs)); + // reconnect as user "foo" and verify that we get permission errors + EXPECT_ZERO(hdfsSingleNameNodeConnect(tlhCluster, &fs, "foo")); + EXPECT_NEGATIVE_ONE_WITH_ERRNO(hdfsChown(fs, paths.file1, "ha3", NULL), EACCES); + EXPECT_ZERO(hdfsDisconnect(fs)); + // reconnect to do the final delete. + EXPECT_ZERO(hdfsSingleNameNodeConnect(tlhCluster, &fs, NULL)); + EXPECT_ZERO(hdfsDelete(fs, paths.prefix, 1)); + EXPECT_ZERO(hdfsDisconnect(fs)); return 0; } static void *testHdfsOperations(void *v) { struct tlhThreadInfo *ti = (struct tlhThreadInfo*)v; - hdfsFS fs = NULL; - int ret; - - fprintf(stderr, "testHdfsOperations(threadIdx=%d): starting\n", - ti->threadIdx); - ret = hdfsSingleNameNodeConnect(tlhCluster, &fs); - if (ret) { - fprintf(stderr, "testHdfsOperations(threadIdx=%d): " - "hdfsSingleNameNodeConnect failed with error %d.\n", - ti->threadIdx, ret); - ti->success = EIO; - return NULL; - } - ti->success = doTestHdfsOperations(ti, fs); - if (hdfsDisconnect(fs)) { - ret = errno; - fprintf(stderr, "hdfsDisconnect error %d\n", ret); - ti->success = ret; - } + int ret = testHdfsOperationsImpl(ti); + ti->success = ret; return NULL; } From 5dabc227657c29ea21528674d9ca1aa0e4c2bdd4 Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Tue, 3 Dec 2013 23:21:30 +0000 Subject: [PATCH 07/32] YARN-1332. In TestAMRMClient, replace assertTrue with assertEquals where possible (Sebastian Wong via Sandy Ryza) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1547640 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../yarn/client/api/impl/TestAMRMClient.java | 122 +++++++++--------- 2 files changed, 64 insertions(+), 61 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index adbcdd00500..6838b8e9fa6 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -132,6 +132,9 @@ Release 2.4.0 - UNRELEASED YARN-1318. Promoted AdminService to an Always-On service and merged it into RMHAProtocolService. (Karthik Kambatla via vinodkv) + YARN-1332. In TestAMRMClient, replace assertTrue with assertEquals where + possible (Sebastian Wong via Sandy Ryza) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java index 1f7565be181..08e71c1cb84 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java @@ -248,7 +248,7 @@ public class TestAMRMClient { matches = amClient.getMatchingRequests(priority, node, testCapability1); verifyMatches(matches, 1); storedRequest = matches.get(0).iterator().next(); - assertTrue(storedContainer1 == storedRequest); + assertEquals(storedContainer1, storedRequest); amClient.removeContainerRequest(storedContainer1); // exact matching with order maintained @@ -259,9 +259,9 @@ public class TestAMRMClient { int i = 0; for(ContainerRequest storedRequest1 : matches.get(0)) { if(i++ == 0) { - assertTrue(storedContainer4 == storedRequest1); + assertEquals(storedContainer4, storedRequest1); } else { - assertTrue(storedContainer6 == storedRequest1); + assertEquals(storedContainer6, storedRequest1); } } amClient.removeContainerRequest(storedContainer6); @@ -276,7 +276,7 @@ public class TestAMRMClient { assert(matches.size() == 2); // verify non-fitting containers are not returned and fitting ones are for(Collection testSet : matches) { - assertTrue(testSet.size() == 1); + assertEquals(1, testSet.size()); ContainerRequest testRequest = testSet.iterator().next(); assertTrue(testRequest != storedContainer4); assertTrue(testRequest != storedContainer5); @@ -310,8 +310,8 @@ public class TestAMRMClient { private void verifyMatches( List> matches, int matchSize) { - assertTrue(matches.size() == 1); - assertTrue(matches.get(0).size() == matchSize); + assertEquals(1, matches.size()); + assertEquals(matches.get(0).size(), matchSize); } @Test (timeout=60000) @@ -337,12 +337,12 @@ public class TestAMRMClient { matches = amClient.getMatchingRequests(priority, node, capability); verifyMatches(matches, 1); storedRequest = matches.get(0).iterator().next(); - assertTrue(storedContainer1 == storedRequest); + assertEquals(storedContainer1, storedRequest); // inferred match rack matches = amClient.getMatchingRequests(priority, rack, capability); verifyMatches(matches, 1); storedRequest = matches.get(0).iterator().next(); - assertTrue(storedContainer1 == storedRequest); + assertEquals(storedContainer1, storedRequest); // inferred rack match no longer valid after request is removed amClient.removeContainerRequest(storedContainer1); @@ -387,10 +387,10 @@ public class TestAMRMClient { // test addition and storage int containersRequestedAny = amClient.remoteRequestsTable.get(priority) .get(ResourceRequest.ANY).get(capability).remoteRequest.getNumContainers(); - assertTrue(containersRequestedAny == 2); + assertEquals(2, containersRequestedAny); containersRequestedAny = amClient.remoteRequestsTable.get(priority1) .get(ResourceRequest.ANY).get(capability).remoteRequest.getNumContainers(); - assertTrue(containersRequestedAny == 1); + assertEquals(1, containersRequestedAny); List> matches = amClient.getMatchingRequests(priority, node, capability); verifyMatches(matches, 2); @@ -417,7 +417,7 @@ public class TestAMRMClient { // test matching of containers ContainerRequest storedRequest = matches.get(0).iterator().next(); - assertTrue(storedContainer1 == storedRequest); + assertEquals(storedContainer1, storedRequest); amClient.removeContainerRequest(storedContainer1); matches = amClient.getMatchingRequests(priority, ResourceRequest.ANY, capability); @@ -438,10 +438,10 @@ public class TestAMRMClient { && iterationsLeft-- > 0) { Log.info(" == alloc " + allocatedContainerCount + " it left " + iterationsLeft); AllocateResponse allocResponse = amClient.allocate(0.1f); - assertTrue(amClient.ask.size() == 0); - assertTrue(amClient.release.size() == 0); + assertEquals(0, amClient.ask.size()); + assertEquals(0, amClient.release.size()); - assertTrue(nodeCount == amClient.getClusterNodeCount()); + assertEquals(nodeCount, amClient.getClusterNodeCount()); allocatedContainerCount += allocResponse.getAllocatedContainers().size(); for(Container container : allocResponse.getAllocatedContainers()) { ContainerRequest expectedRequest = @@ -453,7 +453,7 @@ public class TestAMRMClient { // test correct matched container is returned verifyMatches(matches, 1); ContainerRequest matchedRequest = matches.get(0).iterator().next(); - assertTrue(matchedRequest == expectedRequest); + assertEquals(matchedRequest, expectedRequest); amClient.removeContainerRequest(matchedRequest); // assign this container, use it and release it amClient.releaseAssignedContainer(container.getId()); @@ -464,11 +464,11 @@ public class TestAMRMClient { } } - assertTrue(allocatedContainerCount == 2); + assertEquals(2, allocatedContainerCount); AllocateResponse allocResponse = amClient.allocate(0.1f); - assertTrue(amClient.release.size() == 0); - assertTrue(amClient.ask.size() == 0); - assertTrue(allocResponse.getAllocatedContainers().size() == 0); + assertEquals(0, amClient.release.size()); + assertEquals(0, amClient.ask.size()); + assertEquals(0, allocResponse.getAllocatedContainers().size()); // 0 requests left. everything got cleaned up assertTrue(amClient.remoteRequestsTable.isEmpty()); @@ -494,14 +494,14 @@ public class TestAMRMClient { amClient.start(); amClient.registerApplicationMaster("Host", 10000, ""); - assertTrue(amClient.ask.size() == 0); - assertTrue(amClient.release.size() == 0); + assertEquals(0, amClient.ask.size()); + assertEquals(0, amClient.release.size()); ContainerRequest storedContainer1 = new ContainerRequest(capability, nodes, racks, priority); amClient.addContainerRequest(storedContainer1); - assertTrue(amClient.ask.size() == 3); - assertTrue(amClient.release.size() == 0); + assertEquals(3, amClient.ask.size()); + assertEquals(0, amClient.release.size()); List localNodeBlacklist = new ArrayList(); localNodeBlacklist.add(node); @@ -512,7 +512,7 @@ public class TestAMRMClient { int allocatedContainerCount = getAllocatedContainersNumber(amClient, DEFAULT_ITERATION); // the only node is in blacklist, so no allocation - assertTrue(allocatedContainerCount == 0); + assertEquals(0, allocatedContainerCount); // Remove node from blacklist, so get assigned with 2 amClient.updateBlacklist(null, localNodeBlacklist); @@ -521,7 +521,7 @@ public class TestAMRMClient { amClient.addContainerRequest(storedContainer2); allocatedContainerCount = getAllocatedContainersNumber(amClient, DEFAULT_ITERATION); - assertEquals(allocatedContainerCount, 2); + assertEquals(2, allocatedContainerCount); // Test in case exception in allocate(), blacklist is kept assertTrue(amClient.blacklistAdditions.isEmpty()); @@ -538,7 +538,7 @@ public class TestAMRMClient { amClient.allocate(0.1f); fail("there should be an exception here."); } catch (Exception e) { - assertEquals(amClient.blacklistAdditions.size(), 1); + assertEquals(1, amClient.blacklistAdditions.size()); } } finally { if (amClient != null && amClient.getServiceState() == STATE.STARTED) { @@ -565,16 +565,16 @@ public class TestAMRMClient { nodeList01.add(nodes[0]); nodeList01.add(nodes[1]); amClient.updateBlacklist(nodeList01, null); - assertEquals(amClient.blacklistAdditions.size(),2); - assertEquals(amClient.blacklistRemovals.size(),0); + assertEquals(2, amClient.blacklistAdditions.size()); + assertEquals(0, amClient.blacklistRemovals.size()); // Add nodes[0] again, verify it is not added duplicated. List nodeList02 = new ArrayList(); nodeList02.add(nodes[0]); nodeList02.add(nodes[2]); amClient.updateBlacklist(nodeList02, null); - assertEquals(amClient.blacklistAdditions.size(),3); - assertEquals(amClient.blacklistRemovals.size(),0); + assertEquals(3, amClient.blacklistAdditions.size()); + assertEquals(0, amClient.blacklistRemovals.size()); // Add nodes[1] and nodes[2] to removal list, // Verify addition list remove these two nodes. @@ -582,16 +582,16 @@ public class TestAMRMClient { nodeList12.add(nodes[1]); nodeList12.add(nodes[2]); amClient.updateBlacklist(null, nodeList12); - assertEquals(amClient.blacklistAdditions.size(),1); - assertEquals(amClient.blacklistRemovals.size(),2); + assertEquals(1, amClient.blacklistAdditions.size()); + assertEquals(2, amClient.blacklistRemovals.size()); // Add nodes[1] again to addition list, // Verify removal list will remove this node. List nodeList1 = new ArrayList(); nodeList1.add(nodes[1]); amClient.updateBlacklist(nodeList1, null); - assertEquals(amClient.blacklistAdditions.size(),2); - assertEquals(amClient.blacklistRemovals.size(),1); + assertEquals(2, amClient.blacklistAdditions.size()); + assertEquals(1, amClient.blacklistRemovals.size()); } finally { if (amClient != null && amClient.getServiceState() == STATE.STARTED) { amClient.stop(); @@ -606,10 +606,10 @@ public class TestAMRMClient { while (iterationsLeft-- > 0) { Log.info(" == alloc " + allocatedContainerCount + " it left " + iterationsLeft); AllocateResponse allocResponse = amClient.allocate(0.1f); - assertTrue(amClient.ask.size() == 0); - assertTrue(amClient.release.size() == 0); + assertEquals(0, amClient.ask.size()); + assertEquals(0, amClient.release.size()); - assertTrue(nodeCount == amClient.getClusterNodeCount()); + assertEquals(nodeCount, amClient.getClusterNodeCount()); allocatedContainerCount += allocResponse.getAllocatedContainers().size(); if(allocatedContainerCount == 0) { @@ -654,8 +654,8 @@ public class TestAMRMClient { throws YarnException, IOException { // setup container request - assertTrue(amClient.ask.size() == 0); - assertTrue(amClient.release.size() == 0); + assertEquals(0, amClient.ask.size()); + assertEquals(0, amClient.release.size()); amClient.addContainerRequest( new ContainerRequest(capability, nodes, racks, priority)); @@ -677,11 +677,11 @@ public class TestAMRMClient { int containersRequestedAny = amClient.remoteRequestsTable.get(priority) .get(ResourceRequest.ANY).get(capability).remoteRequest.getNumContainers(); - assertTrue(containersRequestedNode == 2); - assertTrue(containersRequestedRack == 2); - assertTrue(containersRequestedAny == 2); - assertTrue(amClient.ask.size() == 3); - assertTrue(amClient.release.size() == 0); + assertEquals(2, containersRequestedNode); + assertEquals(2, containersRequestedRack); + assertEquals(2, containersRequestedAny); + assertEquals(3, amClient.ask.size()); + assertEquals(0, amClient.release.size()); // RM should allocate container within 2 calls to allocate() int allocatedContainerCount = 0; @@ -695,10 +695,10 @@ public class TestAMRMClient { while (allocatedContainerCount < containersRequestedAny && iterationsLeft-- > 0) { AllocateResponse allocResponse = amClient.allocate(0.1f); - assertTrue(amClient.ask.size() == 0); - assertTrue(amClient.release.size() == 0); + assertEquals(0, amClient.ask.size()); + assertEquals(0, amClient.release.size()); - assertTrue(nodeCount == amClient.getClusterNodeCount()); + assertEquals(nodeCount, amClient.getClusterNodeCount()); allocatedContainerCount += allocResponse.getAllocatedContainers().size(); for(Container container : allocResponse.getAllocatedContainers()) { ContainerId rejectContainerId = container.getId(); @@ -724,19 +724,19 @@ public class TestAMRMClient { Assert.assertTrue(receivedNMTokens.size() > 0 && receivedNMTokens.size() <= nodeCount); - assertTrue(allocatedContainerCount == containersRequestedAny); - assertTrue(amClient.release.size() == 2); - assertTrue(amClient.ask.size() == 0); + assertEquals(allocatedContainerCount, containersRequestedAny); + assertEquals(2, amClient.release.size()); + assertEquals(0, amClient.ask.size()); // need to tell the AMRMClient that we dont need these resources anymore amClient.removeContainerRequest( new ContainerRequest(capability, nodes, racks, priority)); amClient.removeContainerRequest( new ContainerRequest(capability, nodes, racks, priority)); - assertTrue(amClient.ask.size() == 3); + assertEquals(3, amClient.ask.size()); // send 0 container count request for resources that are no longer needed ResourceRequest snoopRequest = amClient.ask.iterator().next(); - assertTrue(snoopRequest.getNumContainers() == 0); + assertEquals(0, snoopRequest.getNumContainers()); // test RPC exception handling amClient.addContainerRequest(new ContainerRequest(capability, nodes, @@ -744,7 +744,7 @@ public class TestAMRMClient { amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority)); snoopRequest = amClient.ask.iterator().next(); - assertTrue(snoopRequest.getNumContainers() == 2); + assertEquals(2, snoopRequest.getNumContainers()); ApplicationMasterProtocol realRM = amClient.rmClient; try { @@ -768,12 +768,12 @@ public class TestAMRMClient { amClient.rmClient = realRM; } - assertTrue(amClient.release.size() == 2); - assertTrue(amClient.ask.size() == 3); + assertEquals(2, amClient.release.size()); + assertEquals(3, amClient.ask.size()); snoopRequest = amClient.ask.iterator().next(); // verify that the remove request made in between makeRequest and allocate // has not been lost - assertTrue(snoopRequest.getNumContainers() == 0); + assertEquals(0, snoopRequest.getNumContainers()); iterationsLeft = 3; // do a few iterations to ensure RM is not going send new containers @@ -781,13 +781,13 @@ public class TestAMRMClient { // inform RM of rejection AllocateResponse allocResponse = amClient.allocate(0.1f); // RM did not send new containers because AM does not need any - assertTrue(allocResponse.getAllocatedContainers().size() == 0); + assertEquals(0, allocResponse.getAllocatedContainers().size()); if(allocResponse.getCompletedContainersStatuses().size() > 0) { for(ContainerStatus cStatus :allocResponse .getCompletedContainersStatuses()) { if(releases.contains(cStatus.getContainerId())) { - assertTrue(cStatus.getState() == ContainerState.COMPLETE); - assertTrue(cStatus.getExitStatus() == -100); + assertEquals(cStatus.getState(), ContainerState.COMPLETE); + assertEquals(-100, cStatus.getExitStatus()); releases.remove(cStatus.getContainerId()); } } @@ -797,8 +797,8 @@ public class TestAMRMClient { sleep(100); } } - assertTrue(amClient.ask.size() == 0); - assertTrue(amClient.release.size() == 0); + assertEquals(0, amClient.ask.size()); + assertEquals(0, amClient.release.size()); } private void sleep(int sleepTime) { From 7fe953ed9099efc0c9b964e70c49c57338dada1b Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Wed, 4 Dec 2013 00:22:12 +0000 Subject: [PATCH 08/32] YARN-1454. Fixed test failure issue with TestRMRestart. Contributed by Karthik Kambatla. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1547651 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../hadoop/yarn/server/resourcemanager/TestRMRestart.java | 2 ++ 2 files changed, 5 insertions(+) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 6838b8e9fa6..50c7cc10795 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -197,6 +197,9 @@ Release 2.4.0 - UNRELEASED YARN-895. Changed RM state-store to not crash immediately if RM restarts while the state-store is down. (Jian He via vinodkv) + YARN-1454. Fixed test failure issue with TestRMRestart. (Karthik Kambatla + via vinodkv) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index 9e145668426..38f8542bde5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -1247,6 +1247,8 @@ public class TestRMRestart { // renewDate before renewing Long renewDateBeforeRenew = allTokensRM2.get(dtId1); try{ + // Sleep for one millisecond to make sure renewDataAfterRenew is greater + Thread.sleep(1); // renew recovered token rm2.getRMDTSecretManager().renewToken(token1, "renewer1"); } catch(Exception e) { From 59a2139093971ed663b58370a03ff408078e8b85 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Wed, 4 Dec 2013 10:49:51 +0000 Subject: [PATCH 09/32] HADOOP-10102. Update commons IO from 2.1 to 2.4 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1547759 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++ hadoop-project/pom.xml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 23e82b79462..a73b182635c 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -393,6 +393,8 @@ Release 2.4.0 - UNRELEASED HADOOP-10127. Add ipc.client.connect.retry.interval to control the frequency of connection retries (Karthik Kambatla via Sandy Ryza) + HADOOP-10102. Update commons IO from 2.1 to 2.4 (Akira Ajisaka via stevel) + OPTIMIZATIONS HADOOP-9748. Reduce blocking on UGI.ensureInitialized (daryn) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index ec6d625a3c0..1a0c7eb11f3 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -467,7 +467,7 @@ commons-io commons-io - 2.1 + 2.4 From f791e291ca39eac6aa0650319e8dd606d15d5804 Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Wed, 4 Dec 2013 20:06:25 +0000 Subject: [PATCH 10/32] HDFS-5555. CacheAdmin commands fail when first listed NameNode is in Standby (jxiang via cmccabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1547895 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../org/apache/hadoop/hdfs/DFSClient.java | 16 +--- .../hdfs/protocol/CacheDirectiveIterator.java | 56 +++++++++++ .../hdfs/protocol/CachePoolIterator.java | 53 +++++++++++ .../hadoop/hdfs/protocol/ClientProtocol.java | 10 +- ...amenodeProtocolServerSideTranslatorPB.java | 45 ++------- .../ClientNamenodeProtocolTranslatorPB.java | 84 ++++------------- .../server/namenode/NameNodeRpcServer.java | 53 +---------- .../server/namenode/TestCacheDirectives.java | 3 +- .../namenode/ha/TestRetryCacheWithHA.java | 92 +++++++++++++++++++ 10 files changed, 250 insertions(+), 165 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolIterator.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 4ac7567a988..c43ef6e6d31 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -421,6 +421,9 @@ Trunk (Unreleased) HDFS-5562. TestCacheDirectives and TestFsDatasetCache should stub out native mlock. (Colin McCabe and Akira Ajisaka via wang) + HDFS-5555. CacheAdmin commands fail when first listed NameNode is in + Standby (jxiang via cmccabe) + Release 2.4.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 11cdb4f26d9..f008878e48e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -109,8 +109,10 @@ import org.apache.hadoop.hdfs.client.ClientMmapManager; import org.apache.hadoop.hdfs.client.HdfsDataInputStream; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry; +import org.apache.hadoop.hdfs.protocol.CacheDirectiveIterator; import org.apache.hadoop.hdfs.protocol.CachePoolEntry; import org.apache.hadoop.hdfs.protocol.CachePoolInfo; +import org.apache.hadoop.hdfs.protocol.CachePoolIterator; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks; import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException; @@ -2324,12 +2326,7 @@ public class DFSClient implements java.io.Closeable { public RemoteIterator listCacheDirectives( CacheDirectiveInfo filter) throws IOException { - checkOpen(); - try { - return namenode.listCacheDirectives(0, filter); - } catch (RemoteException re) { - throw re.unwrapRemoteException(); - } + return new CacheDirectiveIterator(namenode, filter); } public void addCachePool(CachePoolInfo info) throws IOException { @@ -2360,12 +2357,7 @@ public class DFSClient implements java.io.Closeable { } public RemoteIterator listCachePools() throws IOException { - checkOpen(); - try { - return namenode.listCachePools(""); - } catch (RemoteException re) { - throw re.unwrapRemoteException(); - } + return new CachePoolIterator(namenode); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java new file mode 100644 index 00000000000..773a2841321 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.protocol; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.BatchedRemoteIterator; + +/** + * CacheDirectiveIterator is a remote iterator that iterates cache directives. + * It supports retrying in case of namenode failover. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class CacheDirectiveIterator + extends BatchedRemoteIterator { + + private final CacheDirectiveInfo filter; + private final ClientProtocol namenode; + + public CacheDirectiveIterator(ClientProtocol namenode, + CacheDirectiveInfo filter) { + super(Long.valueOf(0)); + this.namenode = namenode; + this.filter = filter; + } + + @Override + public BatchedEntries makeRequest(Long prevKey) + throws IOException { + return namenode.listCacheDirectives(prevKey, filter); + } + + @Override + public Long elementToPrevKey(CacheDirectiveEntry entry) { + return entry.getInfo().getId(); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolIterator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolIterator.java new file mode 100644 index 00000000000..44d6b451742 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolIterator.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.protocol; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.BatchedRemoteIterator; + +/** + * CachePoolIterator is a remote iterator that iterates cache pools. + * It supports retrying in case of namenode failover. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class CachePoolIterator + extends BatchedRemoteIterator { + + private final ClientProtocol namenode; + + public CachePoolIterator(ClientProtocol namenode) { + super(""); + this.namenode = namenode; + } + + @Override + public BatchedEntries makeRequest(String prevKey) + throws IOException { + return namenode.listCachePools(prevKey); + } + + @Override + public String elementToPrevKey(CachePoolEntry entry) { + return entry.getInfo().getPoolName(); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index 8852f818f87..709047ac124 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -28,9 +28,9 @@ import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FsServerDefaults; import org.apache.hadoop.fs.InvalidPathException; import org.apache.hadoop.fs.Options; +import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries; import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.ParentNotDirectoryException; -import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -1134,10 +1134,10 @@ public interface ClientProtocol { * listCacheDirectives. * @param filter Parameters to use to filter the list results, * or null to display all directives visible to us. - * @return A RemoteIterator which returns CacheDirectiveInfo objects. + * @return A batch of CacheDirectiveEntry objects. */ @Idempotent - public RemoteIterator listCacheDirectives( + public BatchedEntries listCacheDirectives( long prevId, CacheDirectiveInfo filter) throws IOException; /** @@ -1175,9 +1175,9 @@ public interface ClientProtocol { * * @param prevPool name of the last pool listed, or the empty string if this is * the first invocation of listCachePools - * @return A RemoteIterator which returns CachePool objects. + * @return A batch of CachePoolEntry objects. */ @Idempotent - public RemoteIterator listCachePools(String prevPool) + public BatchedEntries listCachePools(String prevPool) throws IOException; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java index 6529ca51b8b..478b661677c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java @@ -24,12 +24,9 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FsServerDefaults; +import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries; import org.apache.hadoop.fs.Options.Rename; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.RemoteIterator; -import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry; -import org.apache.hadoop.hdfs.protocol.CachePoolInfo; import org.apache.hadoop.hdfs.protocol.CachePoolEntry; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks; @@ -52,8 +49,6 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AllowS import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AllowSnapshotResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AppendRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AppendResponseProto; -import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolEntryProto; -import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolInfoProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CompleteRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CompleteResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ConcatRequestProto; @@ -109,7 +104,6 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ListCa import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ListCachePoolsResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ListCorruptFileBlocksRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ListCorruptFileBlocksResponseProto; -import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveEntryProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ListCacheDirectivesRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ListCacheDirectivesResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.MetaSaveRequestProto; @@ -176,9 +170,7 @@ import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRespons import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProto; import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenResponseProto; import org.apache.hadoop.security.token.Token; -import org.apache.commons.lang.StringUtils; -import com.google.common.primitives.Shorts; import com.google.protobuf.RpcController; import com.google.protobuf.ServiceException; @@ -1079,21 +1071,13 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements try { CacheDirectiveInfo filter = PBHelper.convert(request.getFilter()); - RemoteIterator iter = - server.listCacheDirectives(request.getPrevId(), filter); + BatchedEntries entries = + server.listCacheDirectives(request.getPrevId(), filter); ListCacheDirectivesResponseProto.Builder builder = ListCacheDirectivesResponseProto.newBuilder(); - long prevId = 0; - while (iter.hasNext()) { - CacheDirectiveEntry entry = iter.next(); - builder.addElements(PBHelper.convert(entry)); - prevId = entry.getInfo().getId(); - } - if (prevId == 0) { - builder.setHasMore(false); - } else { - iter = server.listCacheDirectives(prevId, filter); - builder.setHasMore(iter.hasNext()); + builder.setHasMore(entries.hasMore()); + for (int i=0, n=entries.size(); i iter = + BatchedEntries entries = server.listCachePools(request.getPrevPoolName()); ListCachePoolsResponseProto.Builder responseBuilder = ListCachePoolsResponseProto.newBuilder(); - String prevPoolName = null; - while (iter.hasNext()) { - CachePoolEntry entry = iter.next(); - responseBuilder.addEntries(PBHelper.convert(entry)); - prevPoolName = entry.getInfo().getPoolName(); - } - // fill in hasNext - if (prevPoolName == null) { - responseBuilder.setHasMore(false); - } else { - iter = server.listCachePools(prevPoolName); - responseBuilder.setHasMore(iter.hasNext()); + responseBuilder.setHasMore(entries.hasMore()); + for (int i=0, n=entries.size(); i { - private final CacheDirectiveInfo filter; - - public CacheEntriesIterator(long prevKey, - CacheDirectiveInfo filter) { - super(prevKey); - this.filter = filter; - } - - @Override - public BatchedEntries makeRequest( - Long nextKey) throws IOException { - ListCacheDirectivesResponseProto response; - try { - response = rpcProxy.listCacheDirectives(null, - ListCacheDirectivesRequestProto.newBuilder(). - setPrevId(nextKey). - setFilter(PBHelper.convert(filter)). - build()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } - return new BatchedCacheEntries(response); - } - - @Override - public Long elementToPrevKey(CacheDirectiveEntry element) { - return element.getInfo().getId(); - } - } - @Override - public RemoteIterator + public BatchedEntries listCacheDirectives(long prevId, CacheDirectiveInfo filter) throws IOException { if (filter == null) { filter = new CacheDirectiveInfo.Builder().build(); } - return new CacheEntriesIterator(prevId, filter); + try { + return new BatchedCacheEntries( + rpcProxy.listCacheDirectives(null, + ListCacheDirectivesRequestProto.newBuilder(). + setPrevId(prevId). + setFilter(PBHelper.convert(filter)). + build())); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } } @Override @@ -1164,35 +1139,16 @@ public class ClientNamenodeProtocolTranslatorPB implements } } - private class CachePoolIterator - extends BatchedRemoteIterator { - - public CachePoolIterator(String prevKey) { - super(prevKey); - } - - @Override - public BatchedEntries makeRequest(String prevKey) - throws IOException { - try { - return new BatchedCachePoolEntries( - rpcProxy.listCachePools(null, - ListCachePoolsRequestProto.newBuilder(). - setPrevPoolName(prevKey).build())); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } - } - - @Override - public String elementToPrevKey(CachePoolEntry entry) { - return entry.getInfo().getPoolName(); - } - } - @Override - public RemoteIterator listCachePools(String prevKey) + public BatchedEntries listCachePools(String prevKey) throws IOException { - return new CachePoolIterator(prevKey); + try { + return new BatchedCachePoolEntries( + rpcProxy.listCachePools(null, + ListCachePoolsRequestProto.newBuilder(). + setPrevPoolName(prevKey).build())); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index aa42ec676e9..cb235159a50 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -36,7 +36,6 @@ import java.util.Set; import org.apache.commons.logging.Log; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.BatchedRemoteIterator; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.CreateFlag; @@ -46,8 +45,8 @@ import org.apache.hadoop.fs.InvalidPathException; import org.apache.hadoop.fs.Options; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.ParentNotDirectoryException; -import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.UnresolvedLinkException; +import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.ha.HAServiceStatus; @@ -1251,36 +1250,13 @@ class NameNodeRpcServer implements NamenodeProtocols { namesystem.removeCacheDirective(id); } - private class ServerSideCacheEntriesIterator - extends BatchedRemoteIterator { - - private final CacheDirectiveInfo filter; - - public ServerSideCacheEntriesIterator (Long firstKey, - CacheDirectiveInfo filter) { - super(firstKey); - this.filter = filter; - } - - @Override - public BatchedEntries makeRequest( - Long nextKey) throws IOException { - return namesystem.listCacheDirectives(nextKey, filter); - } - - @Override - public Long elementToPrevKey(CacheDirectiveEntry entry) { - return entry.getInfo().getId(); - } - } - @Override - public RemoteIterator listCacheDirectives(long prevId, + public BatchedEntries listCacheDirectives(long prevId, CacheDirectiveInfo filter) throws IOException { if (filter == null) { filter = new CacheDirectiveInfo.Builder().build(); } - return new ServerSideCacheEntriesIterator(prevId, filter); + return namesystem.listCacheDirectives(prevId, filter); } @Override @@ -1298,28 +1274,9 @@ class NameNodeRpcServer implements NamenodeProtocols { namesystem.removeCachePool(cachePoolName); } - private class ServerSideCachePoolIterator - extends BatchedRemoteIterator { - - public ServerSideCachePoolIterator(String prevKey) { - super(prevKey); - } - - @Override - public BatchedEntries makeRequest(String prevKey) - throws IOException { - return namesystem.listCachePools(prevKey); - } - - @Override - public String elementToPrevKey(CachePoolEntry entry) { - return entry.getInfo().getPoolName(); - } - } - @Override - public RemoteIterator listCachePools(String prevKey) + public BatchedEntries listCachePools(String prevKey) throws IOException { - return new ServerSideCachePoolIterator(prevKey); + return namesystem.listCachePools(prevKey != null ? prevKey : ""); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java index 6fd7881a94b..36492669114 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java @@ -57,6 +57,7 @@ import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry; import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; +import org.apache.hadoop.hdfs.protocol.CacheDirectiveIterator; import org.apache.hadoop.hdfs.protocol.CacheDirectiveStats; import org.apache.hadoop.hdfs.protocol.CachePoolEntry; import org.apache.hadoop.hdfs.protocol.CachePoolInfo; @@ -763,7 +764,7 @@ public class TestCacheDirectives { } // Uncache and check each path in sequence RemoteIterator entries = - nnRpc.listCacheDirectives(0, null); + new CacheDirectiveIterator(nnRpc, null); for (int i=0; i poolNames = new HashSet(poolCount); + for (int i=0; i poolNames = new HashSet(poolCount); + Path path = new Path("/p"); + for (int i=0; i poolNames, int active) throws Exception { + HashSet tmpNames = (HashSet)poolNames.clone(); + RemoteIterator pools = dfs.listCachePools(); + int poolCount = poolNames.size(); + for (int i=0; i poolNames, int active) throws Exception { + HashSet tmpNames = (HashSet)poolNames.clone(); + RemoteIterator directives = dfs.listCacheDirectives(null); + int poolCount = poolNames.size(); + for (int i=0; i Date: Wed, 4 Dec 2013 21:40:57 +0000 Subject: [PATCH 11/32] HDFS-5536. Implement HTTP policy for Namenode and DataNode. Contributed by Haohui Mai. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1547925 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/hadoop/http/HttpConfig.java | 16 +- .../src/main/resources/core-default.xml | 4 +- .../src/site/apt/ClusterSetup.apt.vm | 4 + hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 + .../java/org/apache/hadoop/hdfs/DFSUtil.java | 58 ++++++- .../apache/hadoop/hdfs/HdfsConfiguration.java | 1 - .../hadoop/hdfs/server/datanode/DataNode.java | 69 +++++--- .../datanode/SecureDataNodeStarter.java | 105 ++++++------ .../hdfs/server/namenode/BackupNode.java | 9 +- .../namenode/EditLogFileInputStream.java | 2 +- .../hadoop/hdfs/server/namenode/NameNode.java | 57 +------ .../server/namenode/NameNodeHttpServer.java | 152 +++++++++++------- .../server/namenode/SecondaryNameNode.java | 9 +- .../web/resources/NamenodeWebHdfsMethods.java | 8 +- .../src/main/resources/hdfs-default.xml | 13 ++ .../apache/hadoop/hdfs/MiniDFSCluster.java | 21 ++- .../hdfs/MiniDFSClusterWithNodeGroup.java | 3 +- .../namenode/TestNameNodeHttpServer.java | 127 +++++++++++++++ .../TestValidateConfigurationSettings.java | 12 +- .../hadoop/hdfs/web/TestHttpsFileSystem.java | 3 +- 21 files changed, 446 insertions(+), 233 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpConfig.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpConfig.java index fe3e5ae410a..c533fedf65e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpConfig.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpConfig.java @@ -31,15 +31,25 @@ public class HttpConfig { private static Policy policy; public enum Policy { HTTP_ONLY, - HTTPS_ONLY; + HTTPS_ONLY, + HTTP_AND_HTTPS; public static Policy fromString(String value) { - if (value.equalsIgnoreCase(CommonConfigurationKeysPublic - .HTTP_POLICY_HTTPS_ONLY)) { + if (HTTPS_ONLY.name().equalsIgnoreCase(value)) { return HTTPS_ONLY; + } else if (HTTP_AND_HTTPS.name().equalsIgnoreCase(value)) { + return HTTP_AND_HTTPS; } return HTTP_ONLY; } + + public boolean isHttpEnabled() { + return this == HTTP_ONLY || this == HTTP_AND_HTTPS; + } + + public boolean isHttpsEnabled() { + return this == HTTPS_ONLY || this == HTTP_AND_HTTPS; + } } static { diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index c7671c5451f..dba417341ef 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1133,9 +1133,7 @@ hadoop.ssl.enabled false - Whether to use SSL for the HTTP endpoints. If set to true, the - NameNode, DataNode, ResourceManager, NodeManager, HistoryServer and - MapReduceAppMaster web UIs will be served over HTTPS instead HTTP. + Deprecated. Use dfs.http.policy and yarn.http.policy instead. diff --git a/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm index 01044fdccf6..4857cc797a7 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm @@ -754,6 +754,10 @@ KVNO Timestamp Principal | | | Enable HDFS block access tokens for secure operations. | *-------------------------+-------------------------+------------------------+ | <<>> | | | +| | | This value is deprecated. Use dfs.http.policy | +*-------------------------+-------------------------+------------------------+ +| <<>> | or or | | +| | | HTTPS_ONLY turns off http access | *-------------------------+-------------------------+------------------------+ | <<>> | | | *-------------------------+-------------------------+------------------------+ diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index c43ef6e6d31..72c44d0bb2f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -228,6 +228,9 @@ Trunk (Unreleased) HDFS-5430. Support TTL on CacheDirectives. (wang) + HDFS-5536. Implement HTTP policy for Namenode and DataNode. (Haohui Mai via + jing9) + OPTIMIZATIONS HDFS-5349. DNA_CACHE and DNA_UNCACHE should be by blockId only. (cmccabe) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 6eae1552c20..ea65cbf0fe7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -21,6 +21,7 @@ package org.apache.hadoop.hdfs; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault; +import org.apache.hadoop.http.HttpConfig; /** * This class contains constants for configuration keys used @@ -358,6 +359,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final boolean DFS_SUPPORT_APPEND_DEFAULT = true; public static final String DFS_HTTPS_ENABLE_KEY = "dfs.https.enable"; public static final boolean DFS_HTTPS_ENABLE_DEFAULT = false; + public static final String DFS_HTTP_POLICY_KEY = "dfs.http.policy"; + public static final String DFS_HTTP_POLICY_DEFAULT = HttpConfig.Policy.HTTP_ONLY.name(); public static final String DFS_DEFAULT_CHUNK_VIEW_SIZE_KEY = "dfs.default.chunk.view.size"; public static final int DFS_DEFAULT_CHUNK_VIEW_SIZE_DEFAULT = 32*1024; public static final String DFS_DATANODE_HTTPS_ADDRESS_KEY = "dfs.datanode.https.address"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index e978ddfd06c..bebb97f5801 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hdfs; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BACKUP_ADDRESS_KEY; @@ -65,6 +67,7 @@ import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; @@ -78,6 +81,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSDirectory; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.web.SWebHdfsFileSystem; import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; +import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.http.HttpServer; import org.apache.hadoop.ipc.ProtobufRpcEngine; import org.apache.hadoop.ipc.RPC; @@ -1415,12 +1419,58 @@ public class DFSUtil { defaultKey : DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY; } - public static HttpServer.Builder loadSslConfToHttpServerBuilder( - HttpServer.Builder builder, Configuration sslConf) { + /** + * Get http policy. Http Policy is chosen as follows: + *
    + *
  1. If hadoop.ssl.enabled is set, http endpoints are not started. Only + * https endpoints are started on configured https ports
    2. + *
  2. This configuration is overridden by dfs.https.enable configuration, if + * it is set to true. In that case, both http and https endpoints are stared.
    3. + *
  3. All the above configurations are overridden by dfs.http.policy + * configuration. With this configuration you can set http-only, https-only + * and http-and-https endpoints.
    4. + *
+ * See hdfs-default.xml documentation for more details on each of the above + * configuration settings. + */ + public static HttpConfig.Policy getHttpPolicy(Configuration conf) { + String httpPolicy = conf.get(DFSConfigKeys.DFS_HTTP_POLICY_KEY, + DFSConfigKeys.DFS_HTTP_POLICY_DEFAULT); + + HttpConfig.Policy policy = HttpConfig.Policy.fromString(httpPolicy); + + if (policy == HttpConfig.Policy.HTTP_ONLY) { + boolean httpsEnabled = conf.getBoolean( + DFSConfigKeys.DFS_HTTPS_ENABLE_KEY, + DFSConfigKeys.DFS_HTTPS_ENABLE_DEFAULT); + + boolean hadoopSslEnabled = conf.getBoolean( + CommonConfigurationKeys.HADOOP_SSL_ENABLED_KEY, + CommonConfigurationKeys.HADOOP_SSL_ENABLED_DEFAULT); + + if (hadoopSslEnabled) { + LOG.warn(CommonConfigurationKeys.HADOOP_SSL_ENABLED_KEY + + " is deprecated. Please use " + + DFSConfigKeys.DFS_HTTPS_ENABLE_KEY + "."); + policy = HttpConfig.Policy.HTTPS_ONLY; + } else if (httpsEnabled) { + LOG.warn(DFSConfigKeys.DFS_HTTPS_ENABLE_KEY + + " is deprecated. Please use " + + DFSConfigKeys.DFS_HTTPS_ENABLE_KEY + "."); + policy = HttpConfig.Policy.HTTP_AND_HTTPS; + } + } + + conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, policy.name()); + return policy; + } + + public static HttpServer.Builder loadSslConfToHttpServerBuilder(HttpServer.Builder builder, + Configuration sslConf) { return builder .needsClientAuth( - sslConf.getBoolean(DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_KEY, - DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_DEFAULT)) + sslConf.getBoolean(DFS_CLIENT_HTTPS_NEED_AUTH_KEY, + DFS_CLIENT_HTTPS_NEED_AUTH_DEFAULT)) .keyPassword(sslConf.get("ssl.server.keystore.keypassword")) .keyStore(sslConf.get("ssl.server.keystore.location"), sslConf.get("ssl.server.keystore.password"), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java index 695c9fb78ae..345ceaaa79d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hdfs; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configuration.DeprecationDelta; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index a4834f28286..234a558435a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hdfs.server.datanode; - import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; @@ -65,6 +64,7 @@ import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlo import org.apache.hadoop.hdfs.server.protocol.*; import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; import org.apache.hadoop.hdfs.web.resources.Param; +import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.http.HttpServer; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.ReadaheadPool; @@ -181,9 +181,11 @@ public class DataNode extends Configured private DNConf dnConf; private volatile boolean heartbeatsDisabledForTests = false; private DataStorage storage = null; + private HttpServer infoServer = null; private int infoPort; private int infoSecurePort; + DataNodeMetrics metrics; private InetSocketAddress streamingAddr; @@ -288,7 +290,7 @@ public class DataNode extends Configured * explicitly configured in the given config, then it is determined * via the DNS class. * - * @param config + * @param config configuration * @return the hostname (NB: may not be a FQDN) * @throws UnknownHostException if the dfs.datanode.dns.interface * option is used and the hostname can not be determined @@ -306,39 +308,54 @@ public class DataNode extends Configured return name; } + /** + * @see DFSUtil#getHttpPolicy(org.apache.hadoop.conf.Configuration) + * for information related to the different configuration options and + * Http Policy is decided. + */ private void startInfoServer(Configuration conf) throws IOException { - // create a servlet to serve full-file content + HttpServer.Builder builder = new HttpServer.Builder().setName("datanode") + .setConf(conf).setACL(new AccessControlList(conf.get(DFS_ADMIN, " "))); + + HttpConfig.Policy policy = DFSUtil.getHttpPolicy(conf); InetSocketAddress infoSocAddr = DataNode.getInfoAddr(conf); String infoHost = infoSocAddr.getHostName(); - int tmpInfoPort = infoSocAddr.getPort(); - HttpServer.Builder builder = new HttpServer.Builder().setName("datanode") - .addEndpoint(URI.create("http://" + NetUtils.getHostPortString(infoSocAddr))) - .setFindPort(tmpInfoPort == 0).setConf(conf) - .setACL(new AccessControlList(conf.get(DFS_ADMIN, " "))); - LOG.info("Opened info server at " + infoHost + ":" + tmpInfoPort); - if (conf.getBoolean(DFS_HTTPS_ENABLE_KEY, false)) { + if (policy.isHttpEnabled()) { + if (secureResources == null) { + int port = infoSocAddr.getPort(); + builder.addEndpoint(URI.create("http://" + infoHost + ":" + port)); + if (port == 0) { + builder.setFindPort(true); + } + } else { + // The http socket is created externally using JSVC, we add it in + // directly. + builder.setConnector(secureResources.getListener()); + } + } + + if (policy.isHttpsEnabled()) { InetSocketAddress secInfoSocAddr = NetUtils.createSocketAddr(conf.get( DFS_DATANODE_HTTPS_ADDRESS_KEY, infoHost + ":" + 0)); - builder.addEndpoint(URI.create("https://" - + NetUtils.getHostPortString(secInfoSocAddr))); + Configuration sslConf = new Configuration(false); - sslConf.setBoolean(DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_KEY, conf - .getBoolean(DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_KEY, - DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_DEFAULT)); sslConf.addResource(conf.get( DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY, DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_DEFAULT)); + sslConf.setBoolean(DFS_CLIENT_HTTPS_NEED_AUTH_KEY, conf.getBoolean( + DFS_CLIENT_HTTPS_NEED_AUTH_KEY, DFS_CLIENT_HTTPS_NEED_AUTH_DEFAULT)); DFSUtil.loadSslConfToHttpServerBuilder(builder, sslConf); - if(LOG.isDebugEnabled()) { - LOG.debug("Datanode listening for SSL on " + secInfoSocAddr); + int port = secInfoSocAddr.getPort(); + if (port == 0) { + builder.setFindPort(true); } - infoSecurePort = secInfoSocAddr.getPort(); + builder.addEndpoint(URI.create("https://" + infoHost + ":" + port)); } - this.infoServer = (secureResources == null) ? builder.build() : - builder.setConnector(secureResources.getListener()).build(); + this.infoServer = builder.build(); + this.infoServer.addInternalServlet(null, "/streamFile/*", StreamFile.class); this.infoServer.addInternalServlet(null, "/getFileChecksum/*", FileChecksumServlets.GetServlet.class); @@ -354,9 +371,17 @@ public class DataNode extends Configured WebHdfsFileSystem.PATH_PREFIX + "/*"); } this.infoServer.start(); - this.infoPort = infoServer.getConnectorAddress(0).getPort(); + + int connIdx = 0; + if (policy.isHttpEnabled()) { + infoPort = infoServer.getConnectorAddress(connIdx++).getPort(); + } + + if (policy.isHttpsEnabled()) { + infoSecurePort = infoServer.getConnectorAddress(connIdx).getPort(); + } } - + private void startPlugins(Configuration conf) { plugins = conf.getInstances(DFS_DATANODE_PLUGINS_KEY, ServicePlugin.class); for (ServicePlugin p: plugins) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java index 0fda3060858..23de29027e8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java @@ -16,27 +16,20 @@ */ package org.apache.hadoop.hdfs.server.datanode; -import java.io.IOException; import java.net.InetSocketAddress; import java.net.ServerSocket; import java.nio.channels.ServerSocketChannel; -import java.security.GeneralSecurityException; import org.apache.commons.daemon.Daemon; import org.apache.commons.daemon.DaemonContext; import org.apache.hadoop.conf.Configuration; - import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.http.HttpServer; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.ssl.SSLFactory; import org.mortbay.jetty.Connector; -import org.mortbay.jetty.nio.SelectChannelConnector; -import org.mortbay.jetty.security.SslSocketConnector; - -import javax.net.ssl.SSLServerSocketFactory; import com.google.common.annotations.VisibleForTesting; @@ -65,7 +58,6 @@ public class SecureDataNodeStarter implements Daemon { private String [] args; private SecureResources resources; - private SSLFactory sslFactory; @Override public void init(DaemonContext context) throws Exception { @@ -74,9 +66,7 @@ public class SecureDataNodeStarter implements Daemon { // Stash command-line arguments for regular datanode args = context.getArguments(); - - sslFactory = new SSLFactory(SSLFactory.Mode.SERVER, conf); - resources = getSecureResources(sslFactory, conf); + resources = getSecureResources(conf); } @Override @@ -84,68 +74,65 @@ public class SecureDataNodeStarter implements Daemon { System.err.println("Starting regular datanode initialization"); DataNode.secureMain(args, resources); } - - @Override public void destroy() { - sslFactory.destroy(); - } + @Override public void destroy() {} @Override public void stop() throws Exception { /* Nothing to do */ } + /** + * Acquire privileged resources (i.e., the privileged ports) for the data + * node. The privileged resources consist of the port of the RPC server and + * the port of HTTP (not HTTPS) server. + */ @VisibleForTesting - public static SecureResources getSecureResources(final SSLFactory sslFactory, - Configuration conf) throws Exception { + public static SecureResources getSecureResources(Configuration conf) + throws Exception { + HttpConfig.Policy policy = DFSUtil.getHttpPolicy(conf); + // Obtain secure port for data streaming to datanode InetSocketAddress streamingAddr = DataNode.getStreamingAddr(conf); - int socketWriteTimeout = conf.getInt(DFSConfigKeys.DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY, + int socketWriteTimeout = conf.getInt( + DFSConfigKeys.DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY, HdfsServerConstants.WRITE_TIMEOUT); - + ServerSocket ss = (socketWriteTimeout > 0) ? ServerSocketChannel.open().socket() : new ServerSocket(); ss.bind(streamingAddr, 0); - + // Check that we got the port we need if (ss.getLocalPort() != streamingAddr.getPort()) { - throw new RuntimeException("Unable to bind on specified streaming port in secure " + - "context. Needed " + streamingAddr.getPort() + ", got " + ss.getLocalPort()); + throw new RuntimeException( + "Unable to bind on specified streaming port in secure " + + "context. Needed " + streamingAddr.getPort() + ", got " + + ss.getLocalPort()); } - // Obtain secure listener for web server - Connector listener; - if (HttpConfig.isSecure()) { - try { - sslFactory.init(); - } catch (GeneralSecurityException ex) { - throw new IOException(ex); - } - SslSocketConnector sslListener = new SslSocketConnector() { - @Override - protected SSLServerSocketFactory createFactory() throws Exception { - return sslFactory.createSSLServerSocketFactory(); - } - }; - listener = sslListener; - } else { - listener = HttpServer.createDefaultChannelConnector(); - } - - InetSocketAddress infoSocAddr = DataNode.getInfoAddr(conf); - listener.setHost(infoSocAddr.getHostName()); - listener.setPort(infoSocAddr.getPort()); - // Open listener here in order to bind to port as root - listener.open(); - if (listener.getPort() != infoSocAddr.getPort()) { - throw new RuntimeException("Unable to bind on specified info port in secure " + - "context. Needed " + streamingAddr.getPort() + ", got " + ss.getLocalPort()); - } - System.err.println("Successfully obtained privileged resources (streaming port = " - + ss + " ) (http listener port = " + listener.getConnection() +")"); - - if ((ss.getLocalPort() > 1023 || listener.getPort() > 1023) && - UserGroupInformation.isSecurityEnabled()) { - throw new RuntimeException("Cannot start secure datanode with unprivileged ports"); - } System.err.println("Opened streaming server at " + streamingAddr); - System.err.println("Opened info server at " + infoSocAddr); + + // Bind a port for the web server. The code intends to bind HTTP server to + // privileged port only, as the client can authenticate the server using + // certificates if they are communicating through SSL. + Connector listener = null; + if (policy.isHttpEnabled()) { + listener = HttpServer.createDefaultChannelConnector(); + InetSocketAddress infoSocAddr = DataNode.getInfoAddr(conf); + listener.setHost(infoSocAddr.getHostName()); + listener.setPort(infoSocAddr.getPort()); + // Open listener here in order to bind to port as root + listener.open(); + if (listener.getPort() != infoSocAddr.getPort()) { + throw new RuntimeException("Unable to bind on specified info port in secure " + + "context. Needed " + streamingAddr.getPort() + ", got " + ss.getLocalPort()); + } + System.err.println("Successfully obtained privileged resources (streaming port = " + + ss + " ) (http listener port = " + listener.getConnection() +")"); + + if ((ss.getLocalPort() > 1023 || listener.getPort() > 1023) && + UserGroupInformation.isSecurityEnabled()) { + throw new RuntimeException("Cannot start secure datanode with unprivileged ports"); + } + System.err.println("Opened info server at " + infoSocAddr); + } + return new SecureResources(ss, listener); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java index 44f0f3ef8be..ba65d25fa0b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java @@ -123,11 +123,6 @@ public class BackupNode extends NameNode { String addr = conf.get(BN_HTTP_ADDRESS_NAME_KEY, BN_HTTP_ADDRESS_DEFAULT); return NetUtils.createSocketAddr(addr); } - - @Override // NameNode - protected void setHttpServerAddress(Configuration conf){ - conf.set(BN_HTTP_ADDRESS_NAME_KEY, NetUtils.getHostPortString(getHttpAddress())); - } @Override // NameNode protected void loadNamesystem(Configuration conf) throws IOException { @@ -164,6 +159,10 @@ public class BackupNode extends NameNode { registerWith(nsInfo); // Checkpoint daemon should start after the rpc server started runCheckpointDaemon(conf); + InetSocketAddress addr = getHttpAddress(); + if (addr != null) { + conf.set(BN_HTTP_ADDRESS_NAME_KEY, NetUtils.getHostPortString(getHttpAddress())); + } } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java index ab1634b0c96..494a9dacddf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java @@ -117,7 +117,7 @@ public class EditLogFileInputStream extends EditLogInputStream { */ public static EditLogInputStream fromUrl( URLConnectionFactory connectionFactory, URL url, long startTxId, - long endTxId, boolean inProgress) { + long endTxId, boolean inProgress) { return new EditLogFileInputStream(new URLLog(connectionFactory, url), startTxId, endTxId, inProgress); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index ea07b065926..3b39bc3de99 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -17,6 +17,10 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; + import java.io.IOException; import java.io.PrintStream; import java.net.InetSocketAddress; @@ -430,17 +434,11 @@ public class NameNode implements NameNodeStatusMXBean { return getHttpAddress(conf); } - /** @return the NameNode HTTP address set in the conf. */ + /** @return the NameNode HTTP address. */ public static InetSocketAddress getHttpAddress(Configuration conf) { return NetUtils.createSocketAddr( conf.get(DFS_NAMENODE_HTTP_ADDRESS_KEY, DFS_NAMENODE_HTTP_ADDRESS_DEFAULT)); } - - protected void setHttpServerAddress(Configuration conf) { - String hostPort = NetUtils.getHostPortString(getHttpAddress()); - conf.set(DFS_NAMENODE_HTTP_ADDRESS_KEY, hostPort); - LOG.info("Web-server up at: " + hostPort); - } protected void loadNamesystem(Configuration conf) throws IOException { this.namesystem = FSNamesystem.loadFromDisk(conf); @@ -490,7 +488,6 @@ public class NameNode implements NameNodeStatusMXBean { if (NamenodeRole.NAMENODE == role) { startHttpServer(conf); - validateConfigurationSettingsOrAbort(conf); } loadNamesystem(conf); @@ -498,8 +495,6 @@ public class NameNode implements NameNodeStatusMXBean { if (NamenodeRole.NAMENODE == role) { httpServer.setNameNodeAddress(getNameNodeAddress()); httpServer.setFSImage(getFSImage()); - } else { - validateConfigurationSettingsOrAbort(conf); } pauseMonitor = new JvmPauseMonitor(conf); @@ -517,45 +512,6 @@ public class NameNode implements NameNodeStatusMXBean { return new NameNodeRpcServer(conf, this); } - /** - * Verifies that the final Configuration Settings look ok for the NameNode to - * properly start up - * Things to check for include: - * - HTTP Server Port does not equal the RPC Server Port - * @param conf - * @throws IOException - */ - protected void validateConfigurationSettings(final Configuration conf) - throws IOException { - // check to make sure the web port and rpc port do not match - if(getHttpServerAddress(conf).getPort() - == getRpcServerAddress(conf).getPort()) { - String errMsg = "dfs.namenode.rpc-address " + - "("+ getRpcServerAddress(conf) + ") and " + - "dfs.namenode.http-address ("+ getHttpServerAddress(conf) + ") " + - "configuration keys are bound to the same port, unable to start " + - "NameNode. Port: " + getRpcServerAddress(conf).getPort(); - throw new IOException(errMsg); - } - } - - /** - * Validate NameNode configuration. Log a fatal error and abort if - * configuration is invalid. - * - * @param conf Configuration to validate - * @throws IOException thrown if conf is invalid - */ - private void validateConfigurationSettingsOrAbort(Configuration conf) - throws IOException { - try { - validateConfigurationSettings(conf); - } catch (IOException e) { - LOG.fatal(e.toString()); - throw e; - } - } - /** Start the services common to active and standby states */ private void startCommonServices(Configuration conf) throws IOException { namesystem.startCommonServices(conf, haContext); @@ -634,7 +590,6 @@ public class NameNode implements NameNodeStatusMXBean { httpServer = new NameNodeHttpServer(conf, this, getHttpServerAddress(conf)); httpServer.start(); httpServer.setStartupProgress(startupProgress); - setHttpServerAddress(conf); } private void stopHttpServer() { @@ -656,7 +611,7 @@ public class NameNode implements NameNodeStatusMXBean { *
  • {@link StartupOption#CHECKPOINT CHECKPOINT} - start checkpoint node
    • *
  • {@link StartupOption#UPGRADE UPGRADE} - start the cluster * upgrade and create a snapshot of the current file system state
    • - *
  • {@link StartupOption#RECOVERY RECOVERY} - recover name node + *
  • {@link StartupOption#RECOVER RECOVERY} - recover name node * metadata
    • *
  • {@link StartupOption#ROLLBACK ROLLBACK} - roll the * cluster back to the previous state
    • diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java index b2072a648ad..b72d546d14e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hdfs.server.namenode; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ADMIN; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_KEY; import java.io.IOException; import java.net.InetSocketAddress; @@ -37,6 +39,7 @@ import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMetho import org.apache.hadoop.hdfs.web.AuthFilter; import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; import org.apache.hadoop.hdfs.web.resources.Param; +import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.http.HttpServer; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.SecurityUtil; @@ -60,48 +63,15 @@ public class NameNodeHttpServer { public static final String FSIMAGE_ATTRIBUTE_KEY = "name.system.image"; protected static final String NAMENODE_ATTRIBUTE_KEY = "name.node"; public static final String STARTUP_PROGRESS_ATTRIBUTE_KEY = "startup.progress"; - - public NameNodeHttpServer( - Configuration conf, - NameNode nn, + + NameNodeHttpServer(Configuration conf, NameNode nn, InetSocketAddress bindAddress) { this.conf = conf; this.nn = nn; this.bindAddress = bindAddress; } - - void start() throws IOException { - final String infoHost = bindAddress.getHostName(); - int infoPort = bindAddress.getPort(); - HttpServer.Builder builder = new HttpServer.Builder().setName("hdfs") - .addEndpoint(URI.create(("http://" + NetUtils.getHostPortString(bindAddress)))) - .setFindPort(infoPort == 0).setConf(conf).setACL( - new AccessControlList(conf.get(DFS_ADMIN, " "))) - .setSecurityEnabled(UserGroupInformation.isSecurityEnabled()) - .setUsernameConfKey( - DFSConfigKeys.DFS_NAMENODE_INTERNAL_SPNEGO_USER_NAME_KEY) - .setKeytabConfKey(DFSUtil.getSpnegoKeytabKey(conf, - DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY)); - boolean certSSL = conf.getBoolean(DFSConfigKeys.DFS_HTTPS_ENABLE_KEY, false); - if (certSSL) { - httpsAddress = NetUtils.createSocketAddr(conf.get( - DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, - DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT)); - - builder.addEndpoint(URI.create("https://" - + NetUtils.getHostPortString(httpsAddress))); - Configuration sslConf = new Configuration(false); - sslConf.setBoolean(DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_KEY, conf - .getBoolean(DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_KEY, - DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_DEFAULT)); - sslConf.addResource(conf.get( - DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY, - DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_DEFAULT)); - DFSUtil.loadSslConfToHttpServerBuilder(builder, sslConf); - } - - httpServer = builder.build(); + private void initWebHdfs(Configuration conf) throws IOException { if (WebHdfsFileSystem.isEnabled(conf, HttpServer.LOG)) { //add SPNEGO authentication filter for webhdfs final String name = "SPNEGO"; @@ -115,21 +85,94 @@ public class NameNodeHttpServer { // add webhdfs packages httpServer.addJerseyResourcePackage( NamenodeWebHdfsMethods.class.getPackage().getName() - + ";" + Param.class.getPackage().getName(), pathSpec); + + ";" + Param.class.getPackage().getName(), pathSpec); + } + } + + /** + * @see DFSUtil#getHttpPolicy(org.apache.hadoop.conf.Configuration) + * for information related to the different configuration options and + * Http Policy is decided. + */ + void start() throws IOException { + HttpConfig.Policy policy = DFSUtil.getHttpPolicy(conf); + final String infoHost = bindAddress.getHostName(); + + HttpServer.Builder builder = new HttpServer.Builder() + .setName("hdfs") + .setConf(conf) + .setACL(new AccessControlList(conf.get(DFS_ADMIN, " "))) + .setSecurityEnabled(UserGroupInformation.isSecurityEnabled()) + .setUsernameConfKey( + DFSConfigKeys.DFS_NAMENODE_INTERNAL_SPNEGO_USER_NAME_KEY) + .setKeytabConfKey( + DFSUtil.getSpnegoKeytabKey(conf, + DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY)); + + if (policy.isHttpEnabled()) { + int port = bindAddress.getPort(); + if (port == 0) { + builder.setFindPort(true); } + builder.addEndpoint(URI.create("http://" + infoHost + ":" + port)); + } + + if (policy.isHttpsEnabled()) { + final String httpsAddrString = conf.get( + DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, + DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT); + InetSocketAddress addr = NetUtils.createSocketAddr(httpsAddrString); + + Configuration sslConf = new Configuration(false); + + sslConf.addResource(conf.get( + DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY, + DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_DEFAULT)); + + sslConf.addResource(conf.get( + DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY, + DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_DEFAULT)); + sslConf.setBoolean(DFS_CLIENT_HTTPS_NEED_AUTH_KEY, conf.getBoolean( + DFS_CLIENT_HTTPS_NEED_AUTH_KEY, DFS_CLIENT_HTTPS_NEED_AUTH_DEFAULT)); + DFSUtil.loadSslConfToHttpServerBuilder(builder, sslConf); + + if (addr.getPort() == 0) { + builder.setFindPort(true); + } + + builder.addEndpoint(URI.create("https://" + + NetUtils.getHostPortString(addr))); + } + + httpServer = builder.build(); + + if (policy.isHttpsEnabled()) { + // assume same ssl port for all datanodes + InetSocketAddress datanodeSslPort = NetUtils.createSocketAddr(conf.get( + DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY, infoHost + ":" + + DFSConfigKeys.DFS_DATANODE_HTTPS_DEFAULT_PORT)); + httpServer.setAttribute(DFSConfigKeys.DFS_DATANODE_HTTPS_PORT_KEY, + datanodeSslPort.getPort()); + } + + initWebHdfs(conf); httpServer.setAttribute(NAMENODE_ATTRIBUTE_KEY, nn); httpServer.setAttribute(JspHelper.CURRENT_CONF, conf); setupServlets(httpServer, conf); httpServer.start(); - httpAddress = httpServer.getConnectorAddress(0); - if (certSSL) { - httpsAddress = httpServer.getConnectorAddress(1); - // assume same ssl port for all datanodes - InetSocketAddress datanodeSslPort = NetUtils.createSocketAddr(conf.get( - DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY, infoHost + ":" + 50475)); - httpServer.setAttribute(DFSConfigKeys.DFS_DATANODE_HTTPS_PORT_KEY, datanodeSslPort - .getPort()); + + int connIdx = 0; + if (policy.isHttpEnabled()) { + httpAddress = httpServer.getConnectorAddress(connIdx++); + conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, + NetUtils.getHostPortString(httpAddress)); + } + + if (policy.isHttpsEnabled()) { + httpsAddress = httpServer.getConnectorAddress(connIdx); + conf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, + NetUtils.getHostPortString(httpsAddress)); } } @@ -165,18 +208,17 @@ public class NameNodeHttpServer { return params; } - - public void stop() throws Exception { + void stop() throws Exception { if (httpServer != null) { httpServer.stop(); } } - public InetSocketAddress getHttpAddress() { + InetSocketAddress getHttpAddress() { return httpAddress; } - public InetSocketAddress getHttpsAddress() { + InetSocketAddress getHttpsAddress() { return httpsAddress; } @@ -185,7 +227,7 @@ public class NameNodeHttpServer { * * @param fsImage FSImage to set */ - public void setFSImage(FSImage fsImage) { + void setFSImage(FSImage fsImage) { httpServer.setAttribute(FSIMAGE_ATTRIBUTE_KEY, fsImage); } @@ -194,7 +236,7 @@ public class NameNodeHttpServer { * * @param nameNodeAddress InetSocketAddress to set */ - public void setNameNodeAddress(InetSocketAddress nameNodeAddress) { + void setNameNodeAddress(InetSocketAddress nameNodeAddress) { httpServer.setAttribute(NAMENODE_ADDRESS_ATTRIBUTE_KEY, NetUtils.getConnectAddress(nameNodeAddress)); } @@ -204,7 +246,7 @@ public class NameNodeHttpServer { * * @param prog StartupProgress to set */ - public void setStartupProgress(StartupProgress prog) { + void setStartupProgress(StartupProgress prog) { httpServer.setAttribute(STARTUP_PROGRESS_ATTRIBUTE_KEY, prog); } @@ -234,7 +276,7 @@ public class NameNodeHttpServer { ContentSummaryServlet.class, false); } - public static FSImage getFsImageFromContext(ServletContext context) { + static FSImage getFsImageFromContext(ServletContext context) { return (FSImage)context.getAttribute(FSIMAGE_ATTRIBUTE_KEY); } @@ -242,7 +284,7 @@ public class NameNodeHttpServer { return (NameNode)context.getAttribute(NAMENODE_ATTRIBUTE_KEY); } - public static Configuration getConfFromContext(ServletContext context) { + static Configuration getConfFromContext(ServletContext context) { return (Configuration)context.getAttribute(JspHelper.CURRENT_CONF); } @@ -258,7 +300,7 @@ public class NameNodeHttpServer { * @param context ServletContext to get * @return StartupProgress associated with context */ - public static StartupProgress getStartupProgressFromContext( + static StartupProgress getStartupProgressFromContext( ServletContext context) { return (StartupProgress)context.getAttribute(STARTUP_PROGRESS_ATTRIBUTE_KEY); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java index 022d6e21ac2..05753b1adac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java @@ -30,7 +30,6 @@ import java.io.FilenameFilter; import java.io.IOException; import java.net.InetSocketAddress; import java.net.URI; -import java.net.URISyntaxException; import java.security.PrivilegedAction; import java.security.PrivilegedExceptionAction; import java.util.Collection; @@ -257,12 +256,7 @@ public class SecondaryNameNode implements Runnable { // initialize the webserver for uploading files. int tmpInfoPort = infoSocAddr.getPort(); - URI httpEndpoint; - try { - httpEndpoint = new URI("http://" + NetUtils.getHostPortString(infoSocAddr)); - } catch (URISyntaxException e) { - throw new IOException(e); - } + URI httpEndpoint = URI.create("http://" + NetUtils.getHostPortString(infoSocAddr)); infoServer = new HttpServer.Builder().setName("secondary") .addEndpoint(httpEndpoint) @@ -273,6 +267,7 @@ public class SecondaryNameNode implements Runnable { DFSConfigKeys.DFS_SECONDARY_NAMENODE_INTERNAL_SPNEGO_USER_NAME_KEY) .setKeytabConfKey(DFSUtil.getSpnegoKeytabKey(conf, DFSConfigKeys.DFS_SECONDARY_NAMENODE_KEYTAB_FILE_KEY)).build(); + infoServer.setAttribute("secondary.name.node", this); infoServer.setAttribute("name.system.image", checkpointImage); infoServer.setAttribute(JspHelper.CURRENT_CONF, conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java index 891b0c96e0e..9c7c518ff46 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java @@ -245,8 +245,12 @@ public class NamenodeWebHdfsMethods { + Param.toSortedString("&", parameters); final String uripath = WebHdfsFileSystem.PATH_PREFIX + path; - final URI uri = new URI("http", null, dn.getHostName(), dn.getInfoPort(), - uripath, query, null); + final String scheme = request.getScheme(); + int port = "http".equals(scheme) ? dn.getInfoPort() : dn + .getInfoSecurePort(); + final URI uri = new URI(scheme, null, dn.getHostName(), port, uripath, + query, null); + if (LOG.isTraceEnabled()) { LOG.trace("redirectURI=" + uri); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 6a964fa5219..9eb9983ed16 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -137,7 +137,20 @@ dfs.https.enable false + + Deprecated. Use "dfs.http.policy" instead. + + + + + dfs.http.policy + HTTP_ONLY Decide if HTTPS(SSL) is supported on HDFS + This configures the HTTP endpoint for HDFS daemons: + The following values are supported: + - HTTP_ONLY : Service is provided only on http + - HTTPS_ONLY : Service is provided only on https + - HTTP_AND_HTTPS : Service is provided both on http and https diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index 29c0595a231..72f053c909e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -33,6 +33,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HOSTS; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_DIR_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY; @@ -900,12 +901,17 @@ public class MiniDFSCluster { // After the NN has started, set back the bound ports into // the conf - conf.set(DFSUtil.addKeySuffixes( - DFS_NAMENODE_RPC_ADDRESS_KEY, nameserviceId, nnId), - nn.getNameNodeAddressHostPortString()); - conf.set(DFSUtil.addKeySuffixes( - DFS_NAMENODE_HTTP_ADDRESS_KEY, nameserviceId, nnId), NetUtils - .getHostPortString(nn.getHttpAddress())); + conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, + nameserviceId, nnId), nn.getNameNodeAddressHostPortString()); + if (nn.getHttpAddress() != null) { + conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_HTTP_ADDRESS_KEY, + nameserviceId, nnId), NetUtils.getHostPortString(nn.getHttpAddress())); + } + if (nn.getHttpsAddress() != null) { + conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_HTTPS_ADDRESS_KEY, + nameserviceId, nnId), NetUtils.getHostPortString(nn.getHttpsAddress())); + } + DFSUtil.setGenericConf(conf, nameserviceId, nnId, DFS_NAMENODE_HTTP_ADDRESS_KEY); nameNodes[nnIndex] = new NameNodeInfo(nn, nameserviceId, nnId, @@ -1181,9 +1187,8 @@ public class MiniDFSCluster { SecureResources secureResources = null; if (UserGroupInformation.isSecurityEnabled()) { - SSLFactory sslFactory = new SSLFactory(SSLFactory.Mode.SERVER, dnConf); try { - secureResources = SecureDataNodeStarter.getSecureResources(sslFactory, dnConf); + secureResources = SecureDataNodeStarter.getSecureResources(dnConf); } catch (Exception ex) { ex.printStackTrace(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java index ff8c92a88a3..453ec223fe5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java @@ -158,9 +158,8 @@ public class MiniDFSClusterWithNodeGroup extends MiniDFSCluster { SecureResources secureResources = null; if (UserGroupInformation.isSecurityEnabled()) { - SSLFactory sslFactory = new SSLFactory(SSLFactory.Mode.SERVER, dnConf); try { - secureResources = SecureDataNodeStarter.getSecureResources(sslFactory, dnConf); + secureResources = SecureDataNodeStarter.getSecureResources(dnConf); } catch (Exception ex) { ex.printStackTrace(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java new file mode 100644 index 00000000000..be32b5b2728 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java @@ -0,0 +1,127 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import java.io.File; +import java.net.InetSocketAddress; +import java.net.URL; +import java.net.URLConnection; +import java.util.Arrays; +import java.util.Collection; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.web.URLConnectionFactory; +import org.apache.hadoop.http.HttpConfig; +import org.apache.hadoop.http.HttpConfig.Policy; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.ssl.KeyStoreTestUtil; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +@RunWith(value = Parameterized.class) +public class TestNameNodeHttpServer { + private static final String BASEDIR = System.getProperty("test.build.dir", + "target/test-dir") + "/" + TestNameNodeHttpServer.class.getSimpleName(); + private static String keystoresDir; + private static String sslConfDir; + private static Configuration conf; + private static URLConnectionFactory connectionFactory; + + @Parameters + public static Collection policy() { + Object[][] params = new Object[][] { { HttpConfig.Policy.HTTP_ONLY }, + { HttpConfig.Policy.HTTPS_ONLY }, { HttpConfig.Policy.HTTP_AND_HTTPS } }; + return Arrays.asList(params); + } + + private final HttpConfig.Policy policy; + + public TestNameNodeHttpServer(Policy policy) { + super(); + this.policy = policy; + } + + @BeforeClass + public static void setUp() throws Exception { + File base = new File(BASEDIR); + FileUtil.fullyDelete(base); + base.mkdirs(); + conf = new Configuration(); + keystoresDir = new File(BASEDIR).getAbsolutePath(); + sslConfDir = KeyStoreTestUtil.getClasspathDir(TestNameNodeHttpServer.class); + KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); + connectionFactory = URLConnectionFactory + .newDefaultURLConnectionFactory(conf); + } + + @AfterClass + public static void tearDown() throws Exception { + FileUtil.fullyDelete(new File(BASEDIR)); + KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir); + } + + @Test + public void testHttpPolicy() throws Exception { + conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, policy.name()); + + InetSocketAddress addr = InetSocketAddress.createUnresolved("localhost", 0); + NameNodeHttpServer server = null; + try { + server = new NameNodeHttpServer(conf, null, addr); + server.start(); + + Assert.assertTrue(implies(policy.isHttpEnabled(), + canAccess("http", server.getHttpAddress()))); + Assert.assertTrue(implies(!policy.isHttpEnabled(), + server.getHttpAddress() == null)); + + Assert.assertTrue(implies(policy.isHttpsEnabled(), + canAccess("https", server.getHttpsAddress()))); + Assert.assertTrue(implies(!policy.isHttpsEnabled(), + server.getHttpsAddress() == null)); + + } finally { + server.stop(); + } + } + + private static boolean canAccess(String scheme, InetSocketAddress addr) { + if (addr == null) + return false; + try { + URL url = new URL(scheme + "://" + NetUtils.getHostPortString(addr)); + URLConnection conn = connectionFactory.openConnection(url); + conn.connect(); + conn.getContent(); + } catch (Exception e) { + return false; + } + return true; + } + + private static boolean implies(boolean a, boolean b) { + return !a || b; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestValidateConfigurationSettings.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestValidateConfigurationSettings.java index 54e4f7acbc7..01b0a4f7738 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestValidateConfigurationSettings.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestValidateConfigurationSettings.java @@ -22,6 +22,7 @@ import static org.junit.Assert.fail; import java.io.File; import java.io.IOException; +import java.net.BindException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -50,7 +51,7 @@ public class TestValidateConfigurationSettings { * an exception * is thrown when trying to re-use the same port */ - @Test + @Test(expected = BindException.class) public void testThatMatchingRPCandHttpPortsThrowException() throws IOException { @@ -63,14 +64,7 @@ public class TestValidateConfigurationSettings { FileSystem.setDefaultUri(conf, "hdfs://localhost:9000"); conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "127.0.0.1:9000"); DFSTestUtil.formatNameNode(conf); - try { - NameNode nameNode = new NameNode(conf); - fail("Should have throw the exception since the ports match"); - } catch (IOException e) { - // verify we're getting the right IOException - assertTrue(e.toString().contains("dfs.namenode.rpc-address (")); - System.out.println("Got expected exception: " + e.toString()); - } + new NameNode(conf); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java index 8d2de80fb32..883fdeacf79 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; import org.junit.AfterClass; import org.junit.Assert; @@ -49,7 +50,7 @@ public class TestHttpsFileSystem { public static void setUp() throws Exception { conf = new Configuration(); conf.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true); - conf.setBoolean(DFSConfigKeys.DFS_HTTPS_ENABLE_KEY, true); + conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name()); conf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0"); File base = new File(BASEDIR); From 9ba98e6af1380551c85282e02cbe83e3329228e5 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Eagles Date: Wed, 4 Dec 2013 22:11:28 +0000 Subject: [PATCH 12/32] MAPREDUCE-5632. TestRMContainerAllocator#testUpdatedNodes fails (jeagles) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1547929 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 2 ++ .../mapreduce/v2/app/TestRMContainerAllocator.java | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index a9527bc4bac..fba63583089 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -229,6 +229,8 @@ Release 2.4.0 - UNRELEASED MAPREDUCE-5645. TestFixedLengthInputFormat fails with native libs (Mit Desai via jeagles) + MAPREDUCE-5632. TestRMContainerAllocator#testUpdatedNodes fails (jeagles) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java index ee0544a0537..9a962364e92 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java @@ -101,6 +101,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoSchedule import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.SystemClock; import org.junit.After; +import org.junit.Before; import org.junit.Test; @SuppressWarnings("unchecked") @@ -111,6 +112,12 @@ public class TestRMContainerAllocator { static final RecordFactory recordFactory = RecordFactoryProvider .getRecordFactory(null); + @Before + public void setup() { + MyContainerAllocator.getJobUpdatedNodeEvents().clear(); + MyContainerAllocator.getTaskAttemptKillEvents().clear(); + } + @After public void tearDown() { DefaultMetricsSystem.shutdown(); @@ -770,6 +777,9 @@ public class TestRMContainerAllocator { nm1.nodeHeartbeat(true); dispatcher.await(); + Assert.assertEquals(1, allocator.getJobUpdatedNodeEvents().size()); + Assert.assertEquals(3, allocator.getJobUpdatedNodeEvents().get(0).getUpdatedNodes().size()); + allocator.getJobUpdatedNodeEvents().clear(); // get the assignment assigned = allocator.schedule(); dispatcher.await(); @@ -1501,11 +1511,11 @@ public class TestRMContainerAllocator { return result; } - List getTaskAttemptKillEvents() { + static List getTaskAttemptKillEvents() { return taskAttemptKillEvents; } - List getJobUpdatedNodeEvents() { + static List getJobUpdatedNodeEvents() { return jobUpdatedNodeEvents; } From f5e83a0b3e33376d3378b23f6889d954f4e975f3 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Wed, 4 Dec 2013 22:24:45 +0000 Subject: [PATCH 13/32] HDFS-4983. Numeric usernames do not work with WebHDFS FS. Contributed by Yongjun Zhang. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1547935 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../server/namenode/NameNodeHttpServer.java | 6 ++- .../hadoop/hdfs/web/WebHdfsConfigKeys.java | 38 +++++++++++++++++++ .../hadoop/hdfs/web/WebHdfsFileSystem.java | 2 + .../hadoop/hdfs/web/resources/UserParam.java | 35 ++++++++++++++--- .../apache/hadoop/hdfs/web/TestWebHDFS.java | 29 ++++++++++++++ .../hadoop/hdfs/web/resources/TestParam.java | 15 ++++++++ 7 files changed, 122 insertions(+), 6 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConfigKeys.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 72c44d0bb2f..0d5ae680a5b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -775,6 +775,9 @@ Release 2.3.0 - UNRELEASED HDFS-4997. libhdfs doesn't return correct error codes in most cases (cmccabe) + HDFS-4983. Numeric usernames do not work with WebHDFS FS. + (Yongjun Zhang via wang) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java index b72d546d14e..5e7198d64d4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMetho import org.apache.hadoop.hdfs.web.AuthFilter; import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; import org.apache.hadoop.hdfs.web.resources.Param; +import org.apache.hadoop.hdfs.web.resources.UserParam; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.http.HttpServer; import org.apache.hadoop.net.NetUtils; @@ -73,7 +74,10 @@ public class NameNodeHttpServer { private void initWebHdfs(Configuration conf) throws IOException { if (WebHdfsFileSystem.isEnabled(conf, HttpServer.LOG)) { - //add SPNEGO authentication filter for webhdfs + // set user pattern based on configuration file + UserParam.setUserPattern(conf); + + // add SPNEGO authentication filter for webhdfs final String name = "SPNEGO"; final String classname = AuthFilter.class.getName(); final String pathSpec = WebHdfsFileSystem.PATH_PREFIX + "/*"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConfigKeys.java new file mode 100644 index 00000000000..a4165a08466 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConfigKeys.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.web; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hdfs.DFSConfigKeys; + +/** + * This class contains constants for configuration keys used + * in WebHdfs. + * + */ + +@InterfaceAudience.Private +public class WebHdfsConfigKeys extends DFSConfigKeys { + /** User name pattern key */ + public static final String USER_PATTERN_KEY = + "webhdfs.user.provider.user.pattern"; + /** Default user name pattern value */ + public static final String USER_PATTERN_DEFAULT = + "^[A-Za-z_][A-Za-z0-9._-]*[$]?$"; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index ce4531c55ef..ad948d319f1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -157,6 +157,8 @@ public class WebHdfsFileSystem extends FileSystem ) throws IOException { super.initialize(uri, conf); setConf(conf); + /** set user pattern based on configuration file */ + UserParam.setUserPattern(conf); connectionFactory = URLConnectionFactory .newDefaultURLConnectionFactory(conf); initializeTokenAspect(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserParam.java index 36e128feedb..bdc1cfe55d7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserParam.java @@ -17,8 +17,12 @@ */ package org.apache.hadoop.hdfs.web.resources; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; - +import static org.apache.hadoop.hdfs.web.WebHdfsConfigKeys.USER_PATTERN_KEY; +import static org.apache.hadoop.hdfs.web.WebHdfsConfigKeys.USER_PATTERN_DEFAULT; +import com.google.common.annotations.VisibleForTesting; + import java.text.MessageFormat; import java.util.regex.Pattern; @@ -29,8 +33,29 @@ public class UserParam extends StringParam { /** Default parameter value. */ public static final String DEFAULT = ""; - private static final Domain DOMAIN = new Domain(NAME, - Pattern.compile("^[A-Za-z_][A-Za-z0-9._-]*[$]?$")); + private static String userPattern = null; + private static Domain domain = null; + + static { + setUserPattern(USER_PATTERN_DEFAULT); + } + + @VisibleForTesting + public static String getUserPattern() { + return userPattern; + } + + @VisibleForTesting + public static void setUserPattern(String pattern) { + userPattern = pattern; + Pattern pt = Pattern.compile(userPattern); + domain = new Domain(NAME, pt); + } + + public static void setUserPattern(Configuration conf) { + String pattern = conf.get(USER_PATTERN_KEY, USER_PATTERN_DEFAULT); + setUserPattern(pattern); + } private static String validateLength(String str) { if (str == null) { @@ -50,7 +75,7 @@ public class UserParam extends StringParam { * @param str a string representation of the parameter value. */ public UserParam(final String str) { - super(DOMAIN, str == null || str.equals(DEFAULT)? null : validateLength(str)); + super(domain, str == null || str.equals(DEFAULT)? null : validateLength(str)); } /** @@ -64,4 +89,4 @@ public class UserParam extends StringParam { public String getName() { return NAME; } -} \ No newline at end of file +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java index e49f2268a6a..5dc86a11162 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.TestDFSClientRetries; import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods; +import org.apache.hadoop.hdfs.web.WebHdfsConfigKeys; import org.apache.hadoop.security.UserGroupInformation; import org.apache.log4j.Level; import org.junit.Assert; @@ -261,6 +262,34 @@ public class TestWebHDFS { } } + @Test(timeout=300000) + public void testNumericalUserName() throws Exception { + final Configuration conf = WebHdfsTestUtil.createConf(); + conf.set(WebHdfsConfigKeys.USER_PATTERN_KEY, "^[A-Za-z0-9_][A-Za-z0-9._-]*[$]?$"); + final MiniDFSCluster cluster = + new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + try { + cluster.waitActive(); + WebHdfsTestUtil.getWebHdfsFileSystem(conf, WebHdfsFileSystem.SCHEME) + .setPermission(new Path("/"), + new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); + + UserGroupInformation.createUserForTesting("123", new String[]{"my-group"}) + .doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws IOException, URISyntaxException { + FileSystem fs = WebHdfsTestUtil.getWebHdfsFileSystem(conf, + WebHdfsFileSystem.SCHEME); + Path d = new Path("/my-dir"); + Assert.assertTrue(fs.mkdirs(d)); + return null; + } + }); + } finally { + cluster.shutdown(); + } + } + /** * WebHdfs should be enabled by default after HDFS-5532 * diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java index 97223ab2fb4..b43fc507ca0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java @@ -285,4 +285,19 @@ public class TestParam { Assert.assertEquals(expected, computed.getValue()); } } + + @Test + public void testUserNameOkAfterResettingPattern() { + String oldPattern = UserParam.getUserPattern(); + String newPattern = "^[A-Za-z0-9_][A-Za-z0-9._-]*[$]?$"; + + UserParam.setUserPattern(newPattern); + + UserParam userParam = new UserParam("1x"); + assertNotNull(userParam.getValue()); + userParam = new UserParam("123"); + assertNotNull(userParam.getValue()); + + UserParam.setUserPattern(oldPattern); + } } From 859e425dfa2269ad33f1790e69f3ef189d6485d5 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Thu, 5 Dec 2013 00:05:10 +0000 Subject: [PATCH 14/32] Revert HDFS-4983 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1547970 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 -- .../server/namenode/NameNodeHttpServer.java | 6 +-- .../hadoop/hdfs/web/WebHdfsConfigKeys.java | 38 ------------------- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 2 - .../hadoop/hdfs/web/resources/UserParam.java | 35 +++-------------- .../apache/hadoop/hdfs/web/TestWebHDFS.java | 29 -------------- .../hadoop/hdfs/web/resources/TestParam.java | 15 -------- 7 files changed, 6 insertions(+), 122 deletions(-) delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConfigKeys.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 0d5ae680a5b..72c44d0bb2f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -775,9 +775,6 @@ Release 2.3.0 - UNRELEASED HDFS-4997. libhdfs doesn't return correct error codes in most cases (cmccabe) - HDFS-4983. Numeric usernames do not work with WebHDFS FS. - (Yongjun Zhang via wang) - Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java index 5e7198d64d4..b72d546d14e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java @@ -39,7 +39,6 @@ import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMetho import org.apache.hadoop.hdfs.web.AuthFilter; import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; import org.apache.hadoop.hdfs.web.resources.Param; -import org.apache.hadoop.hdfs.web.resources.UserParam; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.http.HttpServer; import org.apache.hadoop.net.NetUtils; @@ -74,10 +73,7 @@ public class NameNodeHttpServer { private void initWebHdfs(Configuration conf) throws IOException { if (WebHdfsFileSystem.isEnabled(conf, HttpServer.LOG)) { - // set user pattern based on configuration file - UserParam.setUserPattern(conf); - - // add SPNEGO authentication filter for webhdfs + //add SPNEGO authentication filter for webhdfs final String name = "SPNEGO"; final String classname = AuthFilter.class.getName(); final String pathSpec = WebHdfsFileSystem.PATH_PREFIX + "/*"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConfigKeys.java deleted file mode 100644 index a4165a08466..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConfigKeys.java +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hdfs.web; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hdfs.DFSConfigKeys; - -/** - * This class contains constants for configuration keys used - * in WebHdfs. - * - */ - -@InterfaceAudience.Private -public class WebHdfsConfigKeys extends DFSConfigKeys { - /** User name pattern key */ - public static final String USER_PATTERN_KEY = - "webhdfs.user.provider.user.pattern"; - /** Default user name pattern value */ - public static final String USER_PATTERN_DEFAULT = - "^[A-Za-z_][A-Za-z0-9._-]*[$]?$"; -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index ad948d319f1..ce4531c55ef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -157,8 +157,6 @@ public class WebHdfsFileSystem extends FileSystem ) throws IOException { super.initialize(uri, conf); setConf(conf); - /** set user pattern based on configuration file */ - UserParam.setUserPattern(conf); connectionFactory = URLConnectionFactory .newDefaultURLConnectionFactory(conf); initializeTokenAspect(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserParam.java index bdc1cfe55d7..36e128feedb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserParam.java @@ -17,12 +17,8 @@ */ package org.apache.hadoop.hdfs.web.resources; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; -import static org.apache.hadoop.hdfs.web.WebHdfsConfigKeys.USER_PATTERN_KEY; -import static org.apache.hadoop.hdfs.web.WebHdfsConfigKeys.USER_PATTERN_DEFAULT; -import com.google.common.annotations.VisibleForTesting; - + import java.text.MessageFormat; import java.util.regex.Pattern; @@ -33,29 +29,8 @@ public class UserParam extends StringParam { /** Default parameter value. */ public static final String DEFAULT = ""; - private static String userPattern = null; - private static Domain domain = null; - - static { - setUserPattern(USER_PATTERN_DEFAULT); - } - - @VisibleForTesting - public static String getUserPattern() { - return userPattern; - } - - @VisibleForTesting - public static void setUserPattern(String pattern) { - userPattern = pattern; - Pattern pt = Pattern.compile(userPattern); - domain = new Domain(NAME, pt); - } - - public static void setUserPattern(Configuration conf) { - String pattern = conf.get(USER_PATTERN_KEY, USER_PATTERN_DEFAULT); - setUserPattern(pattern); - } + private static final Domain DOMAIN = new Domain(NAME, + Pattern.compile("^[A-Za-z_][A-Za-z0-9._-]*[$]?$")); private static String validateLength(String str) { if (str == null) { @@ -75,7 +50,7 @@ public class UserParam extends StringParam { * @param str a string representation of the parameter value. */ public UserParam(final String str) { - super(domain, str == null || str.equals(DEFAULT)? null : validateLength(str)); + super(DOMAIN, str == null || str.equals(DEFAULT)? null : validateLength(str)); } /** @@ -89,4 +64,4 @@ public class UserParam extends StringParam { public String getName() { return NAME; } -} +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java index 5dc86a11162..e49f2268a6a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java @@ -38,7 +38,6 @@ import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.TestDFSClientRetries; import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods; -import org.apache.hadoop.hdfs.web.WebHdfsConfigKeys; import org.apache.hadoop.security.UserGroupInformation; import org.apache.log4j.Level; import org.junit.Assert; @@ -262,34 +261,6 @@ public class TestWebHDFS { } } - @Test(timeout=300000) - public void testNumericalUserName() throws Exception { - final Configuration conf = WebHdfsTestUtil.createConf(); - conf.set(WebHdfsConfigKeys.USER_PATTERN_KEY, "^[A-Za-z0-9_][A-Za-z0-9._-]*[$]?$"); - final MiniDFSCluster cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); - try { - cluster.waitActive(); - WebHdfsTestUtil.getWebHdfsFileSystem(conf, WebHdfsFileSystem.SCHEME) - .setPermission(new Path("/"), - new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); - - UserGroupInformation.createUserForTesting("123", new String[]{"my-group"}) - .doAs(new PrivilegedExceptionAction() { - @Override - public Void run() throws IOException, URISyntaxException { - FileSystem fs = WebHdfsTestUtil.getWebHdfsFileSystem(conf, - WebHdfsFileSystem.SCHEME); - Path d = new Path("/my-dir"); - Assert.assertTrue(fs.mkdirs(d)); - return null; - } - }); - } finally { - cluster.shutdown(); - } - } - /** * WebHdfs should be enabled by default after HDFS-5532 * diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java index b43fc507ca0..97223ab2fb4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java @@ -285,19 +285,4 @@ public class TestParam { Assert.assertEquals(expected, computed.getValue()); } } - - @Test - public void testUserNameOkAfterResettingPattern() { - String oldPattern = UserParam.getUserPattern(); - String newPattern = "^[A-Za-z0-9_][A-Za-z0-9._-]*[$]?$"; - - UserParam.setUserPattern(newPattern); - - UserParam userParam = new UserParam("1x"); - assertNotNull(userParam.getValue()); - userParam = new UserParam("123"); - assertNotNull(userParam.getValue()); - - UserParam.setUserPattern(oldPattern); - } } From 950e0644b79e0c0514dd036dcf19b9645df0982f Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Thu, 5 Dec 2013 02:59:14 +0000 Subject: [PATCH 15/32] HDFS-5626. dfsadmin report shows incorrect values (cmccabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1548000 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java | 6 +++--- .../apache/hadoop/hdfs/server/datanode/BPServiceActor.java | 2 +- .../org/apache/hadoop/hdfs/server/datanode/DataNode.java | 2 ++ 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 72c44d0bb2f..bc98abd88f3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -427,6 +427,8 @@ Trunk (Unreleased) HDFS-5555. CacheAdmin commands fail when first listed NameNode is in Standby (jxiang via cmccabe) + HDFS-5626. dfsadmin -report shows incorrect cache values. (cmccabe) + Release 2.4.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java index 17636ed146b..2e90de9c254 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java @@ -315,9 +315,9 @@ public class DatanodeInfo extends DatanodeID implements Node { buffer.append("DFS Remaining: " +r+ " ("+StringUtils.byteDesc(r)+")"+"\n"); buffer.append("DFS Used%: "+percent2String(usedPercent) + "\n"); buffer.append("DFS Remaining%: "+percent2String(remainingPercent) + "\n"); - buffer.append("Configured Cache Capacity: "+c+" ("+StringUtils.byteDesc(cc)+")"+"\n"); - buffer.append("Cache Used: "+cu+" ("+StringUtils.byteDesc(u)+")"+"\n"); - buffer.append("Cache Remaining: " +cr+ " ("+StringUtils.byteDesc(r)+")"+"\n"); + buffer.append("Configured Cache Capacity: "+cc+" ("+StringUtils.byteDesc(cc)+")"+"\n"); + buffer.append("Cache Used: "+cu+" ("+StringUtils.byteDesc(cu)+")"+"\n"); + buffer.append("Cache Remaining: " +cr+ " ("+StringUtils.byteDesc(cr)+")"+"\n"); buffer.append("Cache Used%: "+percent2String(cacheUsedPercent) + "\n"); buffer.append("Cache Remaining%: "+percent2String(cacheRemainingPercent) + "\n"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index 1404b24cd96..224a391f1c3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -458,7 +458,7 @@ class BPServiceActor implements Runnable { long createCost = createTime - startTime; long sendCost = sendTime - createTime; dn.getMetrics().addCacheReport(sendCost); - LOG.info("CacheReport of " + blockIds.size() + LOG.debug("CacheReport of " + blockIds.size() + " block(s) took " + createCost + " msec to generate and " + sendCost + " msecs for RPC and NN processing"); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 234a558435a..201e003a359 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -715,6 +715,8 @@ public class DataNode extends Configured ulimit)); } } + LOG.info("Starting DataNode with maxLockedMemory = " + + dnConf.maxLockedMemory); storage = new DataStorage(); From 6d5f8ebed60e59d772e0dcee0b069f8db95f6ccc Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Thu, 5 Dec 2013 03:26:11 +0000 Subject: [PATCH 16/32] YARN-1403. Separate out configuration loading from QueueManager in the Fair Scheduler (Sandy Ryza) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1548006 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../fair/AllocationConfiguration.java | 229 ++++++++ .../fair/AllocationFileLoaderService.java | 398 +++++++++++++ .../scheduler/fair/FSLeafQueue.java | 11 +- .../scheduler/fair/FSParentQueue.java | 13 +- .../scheduler/fair/FSQueue.java | 25 +- .../scheduler/fair/FairScheduler.java | 94 ++- .../fair/FairSchedulerConfiguration.java | 35 +- .../fair/MaxRunningAppsEnforcer.java | 22 +- .../scheduler/fair/QueueManager.java | 540 ++---------------- .../scheduler/fair/QueuePlacementPolicy.java | 32 ++ .../scheduler/fair/SchedulingPolicy.java | 11 +- .../webapp/dao/FairSchedulerQueueInfo.java | 6 +- .../fair/TestAllocationFileLoaderService.java | 432 ++++++++++++++ .../scheduler/fair/TestFSLeafQueue.java | 8 +- .../scheduler/fair/TestFairScheduler.java | 455 ++++----------- .../fair/TestFairSchedulerConfiguration.java | 10 - .../fair/TestMaxRunningAppsEnforcer.java | 15 +- 18 files changed, 1358 insertions(+), 981 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 50c7cc10795..25b712f0866 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -135,6 +135,9 @@ Release 2.4.0 - UNRELEASED YARN-1332. In TestAMRMClient, replace assertTrue with assertEquals where possible (Sebastian Wong via Sandy Ryza) + YARN-1403. Separate out configuration loading from QueueManager in the Fair + Scheduler (Sandy Ryza) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java new file mode 100644 index 00000000000..d12658b63aa --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java @@ -0,0 +1,229 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authorize.AccessControlList; +import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; +import org.apache.hadoop.yarn.util.resource.Resources; + +import com.google.common.annotations.VisibleForTesting; + +public class AllocationConfiguration { + private static final AccessControlList EVERYBODY_ACL = new AccessControlList("*"); + private static final AccessControlList NOBODY_ACL = new AccessControlList(" "); + + // Minimum resource allocation for each queue + private final Map minQueueResources; + // Maximum amount of resources per queue + private final Map maxQueueResources; + // Sharing weights for each queue + private final Map queueWeights; + + // Max concurrent running applications for each queue and for each user; in addition, + // for users that have no max specified, we use the userMaxJobsDefault. + @VisibleForTesting + final Map queueMaxApps; + @VisibleForTesting + final Map userMaxApps; + private final int userMaxAppsDefault; + private final int queueMaxAppsDefault; + + // ACL's for each queue. Only specifies non-default ACL's from configuration. + private final Map> queueAcls; + + // Min share preemption timeout for each queue in seconds. If a job in the queue + // waits this long without receiving its guaranteed share, it is allowed to + // preempt other jobs' tasks. + private final Map minSharePreemptionTimeouts; + + // Default min share preemption timeout for queues where it is not set + // explicitly. + private final long defaultMinSharePreemptionTimeout; + + // Preemption timeout for jobs below fair share in seconds. If a job remains + // below half its fair share for this long, it is allowed to preempt tasks. + private final long fairSharePreemptionTimeout; + + private final Map schedulingPolicies; + + private final SchedulingPolicy defaultSchedulingPolicy; + + // Policy for mapping apps to queues + @VisibleForTesting + QueuePlacementPolicy placementPolicy; + + private final Set queueNames; + + public AllocationConfiguration(Map minQueueResources, + Map maxQueueResources, + Map queueMaxApps, Map userMaxApps, + Map queueWeights, int userMaxAppsDefault, + int queueMaxAppsDefault, Map schedulingPolicies, + SchedulingPolicy defaultSchedulingPolicy, + Map minSharePreemptionTimeouts, + Map> queueAcls, + long fairSharePreemptionTimeout, long defaultMinSharePreemptionTimeout, + QueuePlacementPolicy placementPolicy, Set queueNames) { + this.minQueueResources = minQueueResources; + this.maxQueueResources = maxQueueResources; + this.queueMaxApps = queueMaxApps; + this.userMaxApps = userMaxApps; + this.queueWeights = queueWeights; + this.userMaxAppsDefault = userMaxAppsDefault; + this.queueMaxAppsDefault = queueMaxAppsDefault; + this.defaultSchedulingPolicy = defaultSchedulingPolicy; + this.schedulingPolicies = schedulingPolicies; + this.minSharePreemptionTimeouts = minSharePreemptionTimeouts; + this.queueAcls = queueAcls; + this.fairSharePreemptionTimeout = fairSharePreemptionTimeout; + this.defaultMinSharePreemptionTimeout = defaultMinSharePreemptionTimeout; + this.placementPolicy = placementPolicy; + this.queueNames = queueNames; + } + + public AllocationConfiguration(Configuration conf) { + minQueueResources = new HashMap(); + maxQueueResources = new HashMap(); + queueWeights = new HashMap(); + queueMaxApps = new HashMap(); + userMaxApps = new HashMap(); + userMaxAppsDefault = Integer.MAX_VALUE; + queueMaxAppsDefault = Integer.MAX_VALUE; + queueAcls = new HashMap>(); + minSharePreemptionTimeouts = new HashMap(); + defaultMinSharePreemptionTimeout = Long.MAX_VALUE; + fairSharePreemptionTimeout = Long.MAX_VALUE; + schedulingPolicies = new HashMap(); + defaultSchedulingPolicy = SchedulingPolicy.DEFAULT_POLICY; + placementPolicy = QueuePlacementPolicy.fromConfiguration(conf, + new HashSet()); + queueNames = new HashSet(); + } + + /** + * Get the ACLs associated with this queue. If a given ACL is not explicitly + * configured, include the default value for that ACL. The default for the + * root queue is everybody ("*") and the default for all other queues is + * nobody ("") + */ + public AccessControlList getQueueAcl(String queue, QueueACL operation) { + Map queueAcls = this.queueAcls.get(queue); + if (queueAcls != null) { + AccessControlList operationAcl = queueAcls.get(operation); + if (operationAcl != null) { + return operationAcl; + } + } + return (queue.equals("root")) ? EVERYBODY_ACL : NOBODY_ACL; + } + + /** + * Get a queue's min share preemption timeout, in milliseconds. This is the + * time after which jobs in the queue may kill other queues' tasks if they + * are below their min share. + */ + public long getMinSharePreemptionTimeout(String queueName) { + Long minSharePreemptionTimeout = minSharePreemptionTimeouts.get(queueName); + return (minSharePreemptionTimeout == null) ? defaultMinSharePreemptionTimeout + : minSharePreemptionTimeout; + } + + /** + * Get the fair share preemption, in milliseconds. This is the time + * after which any job may kill other jobs' tasks if it is below half + * its fair share. + */ + public long getFairSharePreemptionTimeout() { + return fairSharePreemptionTimeout; + } + + public ResourceWeights getQueueWeight(String queue) { + ResourceWeights weight = queueWeights.get(queue); + return (weight == null) ? ResourceWeights.NEUTRAL : weight; + } + + public int getUserMaxApps(String user) { + Integer maxApps = userMaxApps.get(user); + return (maxApps == null) ? userMaxAppsDefault : maxApps; + } + + public int getQueueMaxApps(String queue) { + Integer maxApps = queueMaxApps.get(queue); + return (maxApps == null) ? queueMaxAppsDefault : maxApps; + } + + /** + * Get the minimum resource allocation for the given queue. + * @return the cap set on this queue, or 0 if not set. + */ + public Resource getMinResources(String queue) { + Resource minQueueResource = minQueueResources.get(queue); + return (minQueueResource == null) ? Resources.none() : minQueueResource; + } + + /** + * Get the maximum resource allocation for the given queue. + * @return the cap set on this queue, or Integer.MAX_VALUE if not set. + */ + + public Resource getMaxResources(String queueName) { + Resource maxQueueResource = maxQueueResources.get(queueName); + return (maxQueueResource == null) ? Resources.unbounded() : maxQueueResource; + } + + public boolean hasAccess(String queueName, QueueACL acl, + UserGroupInformation user) { + int lastPeriodIndex = queueName.length(); + while (lastPeriodIndex != -1) { + String queue = queueName.substring(0, lastPeriodIndex); + if (getQueueAcl(queue, acl).isUserAllowed(user)) { + return true; + } + + lastPeriodIndex = queueName.lastIndexOf('.', lastPeriodIndex - 1); + } + + return false; + } + + public SchedulingPolicy getSchedulingPolicy(String queueName) { + SchedulingPolicy policy = schedulingPolicies.get(queueName); + return (policy == null) ? defaultSchedulingPolicy : policy; + } + + public SchedulingPolicy getDefaultSchedulingPolicy() { + return defaultSchedulingPolicy; + } + + public Set getQueueNames() { + return queueNames; + } + + public QueuePlacementPolicy getPlacementPolicy() { + return placementPolicy; + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java new file mode 100644 index 00000000000..2a7164d49b8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java @@ -0,0 +1,398 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.authorize.AccessControlList; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; +import org.apache.hadoop.yarn.util.Clock; +import org.apache.hadoop.yarn.util.SystemClock; +import org.apache.hadoop.yarn.util.resource.Resources; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.w3c.dom.Text; +import org.xml.sax.SAXException; + +import com.google.common.annotations.VisibleForTesting; + +@Public +@Unstable +public class AllocationFileLoaderService extends AbstractService { + + public static final Log LOG = LogFactory.getLog( + AllocationFileLoaderService.class.getName()); + + /** Time to wait between checks of the allocation file */ + public static final long ALLOC_RELOAD_INTERVAL_MS = 10 * 1000; + + /** + * Time to wait after the allocation has been modified before reloading it + * (this is done to prevent loading a file that hasn't been fully written). + */ + public static final long ALLOC_RELOAD_WAIT_MS = 5 * 1000; + + private final Clock clock; + + private long lastSuccessfulReload; // Last time we successfully reloaded queues + private boolean lastReloadAttemptFailed = false; + + // Path to XML file containing allocations. + private File allocFile; + + private Listener reloadListener; + + @VisibleForTesting + long reloadIntervalMs = ALLOC_RELOAD_INTERVAL_MS; + + private Thread reloadThread; + private volatile boolean running = true; + + public AllocationFileLoaderService() { + this(new SystemClock()); + } + + public AllocationFileLoaderService(Clock clock) { + super(AllocationFileLoaderService.class.getName()); + this.clock = clock; + + } + + @Override + public void init(Configuration conf) { + this.allocFile = getAllocationFile(conf); + super.init(conf); + } + + @Override + public void start() { + if (allocFile == null) { + return; + } + reloadThread = new Thread() { + public void run() { + while (running) { + long time = clock.getTime(); + long lastModified = allocFile.lastModified(); + if (lastModified > lastSuccessfulReload && + time > lastModified + ALLOC_RELOAD_WAIT_MS) { + try { + reloadAllocations(); + } catch (Exception ex) { + if (!lastReloadAttemptFailed) { + LOG.error("Failed to reload fair scheduler config file - " + + "will use existing allocations.", ex); + } + lastReloadAttemptFailed = true; + } + } else if (lastModified == 0l) { + if (!lastReloadAttemptFailed) { + LOG.warn("Failed to reload fair scheduler config file because" + + " last modified returned 0. File exists: " + allocFile.exists()); + } + lastReloadAttemptFailed = true; + } + try { + Thread.sleep(reloadIntervalMs); + } catch (InterruptedException ex) { + LOG.info("Interrupted while waiting to reload alloc configuration"); + } + } + } + }; + reloadThread.setName("AllocationFileReloader"); + reloadThread.setDaemon(true); + reloadThread.start(); + super.start(); + } + + @Override + public void stop() { + running = false; + reloadThread.interrupt(); + super.stop(); + } + + /** + * Path to XML file containing allocations. If the + * path is relative, it is searched for in the + * classpath, but loaded like a regular File. + */ + public File getAllocationFile(Configuration conf) { + String allocFilePath = conf.get(FairSchedulerConfiguration.ALLOCATION_FILE, + FairSchedulerConfiguration.DEFAULT_ALLOCATION_FILE); + File allocFile = new File(allocFilePath); + if (!allocFile.isAbsolute()) { + URL url = Thread.currentThread().getContextClassLoader() + .getResource(allocFilePath); + if (url == null) { + LOG.warn(allocFilePath + " not found on the classpath."); + allocFile = null; + } else if (!url.getProtocol().equalsIgnoreCase("file")) { + throw new RuntimeException("Allocation file " + url + + " found on the classpath is not on the local filesystem."); + } else { + allocFile = new File(url.getPath()); + } + } + return allocFile; + } + + public synchronized void setReloadListener(Listener reloadListener) { + this.reloadListener = reloadListener; + } + + /** + * Updates the allocation list from the allocation config file. This file is + * expected to be in the XML format specified in the design doc. + * + * @throws IOException if the config file cannot be read. + * @throws AllocationConfigurationException if allocations are invalid. + * @throws ParserConfigurationException if XML parser is misconfigured. + * @throws SAXException if config file is malformed. + */ + public synchronized void reloadAllocations() throws IOException, + ParserConfigurationException, SAXException, AllocationConfigurationException { + if (allocFile == null) { + return; + } + LOG.info("Loading allocation file " + allocFile); + // Create some temporary hashmaps to hold the new allocs, and we only save + // them in our fields if we have parsed the entire allocs file successfully. + Map minQueueResources = new HashMap(); + Map maxQueueResources = new HashMap(); + Map queueMaxApps = new HashMap(); + Map userMaxApps = new HashMap(); + Map queueWeights = new HashMap(); + Map queuePolicies = new HashMap(); + Map minSharePreemptionTimeouts = new HashMap(); + Map> queueAcls = + new HashMap>(); + int userMaxAppsDefault = Integer.MAX_VALUE; + int queueMaxAppsDefault = Integer.MAX_VALUE; + long fairSharePreemptionTimeout = Long.MAX_VALUE; + long defaultMinSharePreemptionTimeout = Long.MAX_VALUE; + SchedulingPolicy defaultSchedPolicy = SchedulingPolicy.DEFAULT_POLICY; + + QueuePlacementPolicy newPlacementPolicy = null; + + // Remember all queue names so we can display them on web UI, etc. + Set queueNamesInAllocFile = new HashSet(); + + // Read and parse the allocations file. + DocumentBuilderFactory docBuilderFactory = + DocumentBuilderFactory.newInstance(); + docBuilderFactory.setIgnoringComments(true); + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + Document doc = builder.parse(allocFile); + Element root = doc.getDocumentElement(); + if (!"allocations".equals(root.getTagName())) + throw new AllocationConfigurationException("Bad fair scheduler config " + + "file: top-level element not "); + NodeList elements = root.getChildNodes(); + List queueElements = new ArrayList(); + Element placementPolicyElement = null; + for (int i = 0; i < elements.getLength(); i++) { + Node node = elements.item(i); + if (node instanceof Element) { + Element element = (Element)node; + if ("queue".equals(element.getTagName()) || + "pool".equals(element.getTagName())) { + queueElements.add(element); + } else if ("user".equals(element.getTagName())) { + String userName = element.getAttribute("name"); + NodeList fields = element.getChildNodes(); + for (int j = 0; j < fields.getLength(); j++) { + Node fieldNode = fields.item(j); + if (!(fieldNode instanceof Element)) + continue; + Element field = (Element) fieldNode; + if ("maxRunningApps".equals(field.getTagName())) { + String text = ((Text)field.getFirstChild()).getData().trim(); + int val = Integer.parseInt(text); + userMaxApps.put(userName, val); + } + } + } else if ("userMaxAppsDefault".equals(element.getTagName())) { + String text = ((Text)element.getFirstChild()).getData().trim(); + int val = Integer.parseInt(text); + userMaxAppsDefault = val; + } else if ("fairSharePreemptionTimeout".equals(element.getTagName())) { + String text = ((Text)element.getFirstChild()).getData().trim(); + long val = Long.parseLong(text) * 1000L; + fairSharePreemptionTimeout = val; + } else if ("defaultMinSharePreemptionTimeout".equals(element.getTagName())) { + String text = ((Text)element.getFirstChild()).getData().trim(); + long val = Long.parseLong(text) * 1000L; + defaultMinSharePreemptionTimeout = val; + } else if ("queueMaxAppsDefault".equals(element.getTagName())) { + String text = ((Text)element.getFirstChild()).getData().trim(); + int val = Integer.parseInt(text); + queueMaxAppsDefault = val; + } else if ("defaultQueueSchedulingPolicy".equals(element.getTagName()) + || "defaultQueueSchedulingMode".equals(element.getTagName())) { + String text = ((Text)element.getFirstChild()).getData().trim(); + defaultSchedPolicy = SchedulingPolicy.parse(text); + } else if ("queuePlacementPolicy".equals(element.getTagName())) { + placementPolicyElement = element; + } else { + LOG.warn("Bad element in allocations file: " + element.getTagName()); + } + } + } + + // Load queue elements. A root queue can either be included or omitted. If + // it's included, all other queues must be inside it. + for (Element element : queueElements) { + String parent = "root"; + if (element.getAttribute("name").equalsIgnoreCase("root")) { + if (queueElements.size() > 1) { + throw new AllocationConfigurationException("If configuring root queue," + + " no other queues can be placed alongside it."); + } + parent = null; + } + loadQueue(parent, element, minQueueResources, maxQueueResources, queueMaxApps, + userMaxApps, queueWeights, queuePolicies, minSharePreemptionTimeouts, + queueAcls, queueNamesInAllocFile); + } + + // Load placement policy and pass it configured queues + Configuration conf = getConfig(); + if (placementPolicyElement != null) { + newPlacementPolicy = QueuePlacementPolicy.fromXml(placementPolicyElement, + queueNamesInAllocFile, conf); + } else { + newPlacementPolicy = QueuePlacementPolicy.fromConfiguration(conf, + queueNamesInAllocFile); + } + + AllocationConfiguration info = new AllocationConfiguration(minQueueResources, maxQueueResources, + queueMaxApps, userMaxApps, queueWeights, userMaxAppsDefault, + queueMaxAppsDefault, queuePolicies, defaultSchedPolicy, minSharePreemptionTimeouts, + queueAcls, fairSharePreemptionTimeout, defaultMinSharePreemptionTimeout, + newPlacementPolicy, queueNamesInAllocFile); + + lastSuccessfulReload = clock.getTime(); + lastReloadAttemptFailed = false; + + reloadListener.onReload(info); + } + + /** + * Loads a queue from a queue element in the configuration file + */ + private void loadQueue(String parentName, Element element, Map minQueueResources, + Map maxQueueResources, Map queueMaxApps, + Map userMaxApps, Map queueWeights, + Map queuePolicies, + Map minSharePreemptionTimeouts, + Map> queueAcls, Set queueNamesInAllocFile) + throws AllocationConfigurationException { + String queueName = element.getAttribute("name"); + if (parentName != null) { + queueName = parentName + "." + queueName; + } + Map acls = + new HashMap(); + NodeList fields = element.getChildNodes(); + boolean isLeaf = true; + + for (int j = 0; j < fields.getLength(); j++) { + Node fieldNode = fields.item(j); + if (!(fieldNode instanceof Element)) + continue; + Element field = (Element) fieldNode; + if ("minResources".equals(field.getTagName())) { + String text = ((Text)field.getFirstChild()).getData().trim(); + Resource val = FairSchedulerConfiguration.parseResourceConfigValue(text); + minQueueResources.put(queueName, val); + } else if ("maxResources".equals(field.getTagName())) { + String text = ((Text)field.getFirstChild()).getData().trim(); + Resource val = FairSchedulerConfiguration.parseResourceConfigValue(text); + maxQueueResources.put(queueName, val); + } else if ("maxRunningApps".equals(field.getTagName())) { + String text = ((Text)field.getFirstChild()).getData().trim(); + int val = Integer.parseInt(text); + queueMaxApps.put(queueName, val); + } else if ("weight".equals(field.getTagName())) { + String text = ((Text)field.getFirstChild()).getData().trim(); + double val = Double.parseDouble(text); + queueWeights.put(queueName, new ResourceWeights((float)val)); + } else if ("minSharePreemptionTimeout".equals(field.getTagName())) { + String text = ((Text)field.getFirstChild()).getData().trim(); + long val = Long.parseLong(text) * 1000L; + minSharePreemptionTimeouts.put(queueName, val); + } else if ("schedulingPolicy".equals(field.getTagName()) + || "schedulingMode".equals(field.getTagName())) { + String text = ((Text)field.getFirstChild()).getData().trim(); + SchedulingPolicy policy = SchedulingPolicy.parse(text); + queuePolicies.put(queueName, policy); + } else if ("aclSubmitApps".equals(field.getTagName())) { + String text = ((Text)field.getFirstChild()).getData(); + acls.put(QueueACL.SUBMIT_APPLICATIONS, new AccessControlList(text)); + } else if ("aclAdministerApps".equals(field.getTagName())) { + String text = ((Text)field.getFirstChild()).getData(); + acls.put(QueueACL.ADMINISTER_QUEUE, new AccessControlList(text)); + } else if ("queue".endsWith(field.getTagName()) || + "pool".equals(field.getTagName())) { + loadQueue(queueName, field, minQueueResources, maxQueueResources, + queueMaxApps, userMaxApps, queueWeights, queuePolicies, + minSharePreemptionTimeouts, + queueAcls, queueNamesInAllocFile); + isLeaf = false; + } + } + if (isLeaf) { + queueNamesInAllocFile.add(queueName); + } + queueAcls.put(queueName, acls); + if (maxQueueResources.containsKey(queueName) && minQueueResources.containsKey(queueName) + && !Resources.fitsIn(minQueueResources.get(queueName), + maxQueueResources.get(queueName))) { + LOG.warn(String.format("Queue %s has max resources %d less than min resources %d", + queueName, maxQueueResources.get(queueName), minQueueResources.get(queueName))); + } + } + + public interface Listener { + public void onReload(AllocationConfiguration info); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index a6fbedbc52d..1257cba1fc2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -46,19 +46,15 @@ public class FSLeafQueue extends FSQueue { private final List nonRunnableAppScheds = new ArrayList(); - private final FairScheduler scheduler; - private final QueueManager queueMgr; private Resource demand = Resources.createResource(0); // Variables used for preemption private long lastTimeAtMinShare; private long lastTimeAtHalfFairShare; - public FSLeafQueue(String name, QueueManager queueMgr, FairScheduler scheduler, + public FSLeafQueue(String name, FairScheduler scheduler, FSParentQueue parent) { - super(name, queueMgr, scheduler, parent); - this.scheduler = scheduler; - this.queueMgr = queueMgr; + super(name, scheduler, parent); this.lastTimeAtMinShare = scheduler.getClock().getTime(); this.lastTimeAtHalfFairShare = scheduler.getClock().getTime(); } @@ -145,7 +141,8 @@ public class FSLeafQueue extends FSQueue { public void updateDemand() { // Compute demand by iterating through apps in the queue // Limit demand to maxResources - Resource maxRes = queueMgr.getMaxResources(getName()); + Resource maxRes = scheduler.getAllocationConfiguration() + .getMaxResources(getName()); demand = Resources.createResource(0); for (AppSchedulable sched : runnableAppScheds) { if (Resources.equals(demand, maxRes)) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java index 45d2811919f..0a9272594c7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java @@ -41,14 +41,12 @@ public class FSParentQueue extends FSQueue { private final List childQueues = new ArrayList(); - private final QueueManager queueMgr; private Resource demand = Resources.createResource(0); private int runnableApps; - public FSParentQueue(String name, QueueManager queueMgr, FairScheduler scheduler, + public FSParentQueue(String name, FairScheduler scheduler, FSParentQueue parent) { - super(name, queueMgr, scheduler, parent); - this.queueMgr = queueMgr; + super(name, scheduler, parent); } public void addChildQueue(FSQueue child) { @@ -82,7 +80,8 @@ public class FSParentQueue extends FSQueue { public void updateDemand() { // Compute demand by iterating through apps in the queue // Limit demand to maxResources - Resource maxRes = queueMgr.getMaxResources(getName()); + Resource maxRes = scheduler.getAllocationConfiguration() + .getMaxResources(getName()); demand = Resources.createResource(0); for (FSQueue childQueue : childQueues) { childQueue.updateDemand(); @@ -164,8 +163,8 @@ public class FSParentQueue extends FSQueue { public void setPolicy(SchedulingPolicy policy) throws AllocationConfigurationException { boolean allowed = - SchedulingPolicy.isApplicableTo(policy, (this == queueMgr - .getRootQueue()) ? SchedulingPolicy.DEPTH_ROOT + SchedulingPolicy.isApplicableTo(policy, (parent == null) + ? SchedulingPolicy.DEPTH_ROOT : SchedulingPolicy.DEPTH_INTERMEDIATE); if (!allowed) { throwPolicyDoesnotApplyException(policy); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java index 94c8f70a9aa..8b3d90d7666 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java @@ -39,20 +39,17 @@ import org.apache.hadoop.yarn.util.resource.Resources; @Unstable public abstract class FSQueue extends Schedulable implements Queue { private final String name; - private final QueueManager queueMgr; - private final FairScheduler scheduler; + protected final FairScheduler scheduler; private final FSQueueMetrics metrics; protected final FSParentQueue parent; protected final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); - protected SchedulingPolicy policy = SchedulingPolicy.getDefault(); + protected SchedulingPolicy policy = SchedulingPolicy.DEFAULT_POLICY; - public FSQueue(String name, QueueManager queueMgr, - FairScheduler scheduler, FSParentQueue parent) { + public FSQueue(String name, FairScheduler scheduler, FSParentQueue parent) { this.name = name; - this.queueMgr = queueMgr; this.scheduler = scheduler; this.metrics = FSQueueMetrics.forQueue(getName(), parent, true, scheduler.getConf()); metrics.setMinShare(getMinShare()); @@ -88,17 +85,17 @@ public abstract class FSQueue extends Schedulable implements Queue { @Override public ResourceWeights getWeights() { - return queueMgr.getQueueWeight(getName()); + return scheduler.getAllocationConfiguration().getQueueWeight(getName()); } @Override public Resource getMinShare() { - return queueMgr.getMinResources(getName()); + return scheduler.getAllocationConfiguration().getMinResources(getName()); } @Override public Resource getMaxShare() { - return queueMgr.getMaxResources(getName()); + return scheduler.getAllocationConfiguration().getMaxResources(getName()); } @Override @@ -148,13 +145,7 @@ public abstract class FSQueue extends Schedulable implements Queue { } public boolean hasAccess(QueueACL acl, UserGroupInformation user) { - // Check if the leaf-queue allows access - if (queueMgr.getQueueAcl(getName(), acl).isUserAllowed(user)) { - return true; - } - - // Check if parent-queue allows access - return parent != null && parent.hasAccess(acl, user); + return scheduler.getAllocationConfiguration().hasAccess(name, acl, user); } /** @@ -181,7 +172,7 @@ public abstract class FSQueue extends Schedulable implements Queue { */ protected boolean assignContainerPreCheck(FSSchedulerNode node) { if (!Resources.fitsIn(getResourceUsage(), - queueMgr.getMaxResources(getName())) + scheduler.getAllocationConfiguration().getMaxResources(getName())) || node.getReservedContainer() != null) { return false; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index a882113c004..a439adc68b6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -192,11 +192,16 @@ public class FairScheduler implements ResourceScheduler { @VisibleForTesting final MaxRunningAppsEnforcer maxRunningEnforcer; + + private AllocationFileLoaderService allocsLoader; + @VisibleForTesting + AllocationConfiguration allocConf; public FairScheduler() { clock = new SystemClock(); + allocsLoader = new AllocationFileLoaderService(); queueMgr = new QueueManager(this); - maxRunningEnforcer = new MaxRunningAppsEnforcer(queueMgr); + maxRunningEnforcer = new MaxRunningAppsEnforcer(this); } private void validateConf(Configuration conf) { @@ -275,7 +280,6 @@ public class FairScheduler implements ResourceScheduler { * required resources per job. */ protected synchronized void update() { - queueMgr.reloadAllocsIfNecessary(); // Relaod alloc file updatePreemptionVariables(); // Determine if any queues merit preemption FSQueue rootQueue = queueMgr.getRootQueue(); @@ -480,8 +484,8 @@ public class FairScheduler implements ResourceScheduler { */ protected Resource resToPreempt(FSLeafQueue sched, long curTime) { String queue = sched.getName(); - long minShareTimeout = queueMgr.getMinSharePreemptionTimeout(queue); - long fairShareTimeout = queueMgr.getFairSharePreemptionTimeout(); + long minShareTimeout = allocConf.getMinSharePreemptionTimeout(queue); + long fairShareTimeout = allocConf.getFairSharePreemptionTimeout(); Resource resDueToMinShare = Resources.none(); Resource resDueToFairShare = Resources.none(); if (curTime - sched.getLastTimeAtMinShare() > minShareTimeout) { @@ -650,8 +654,8 @@ public class FairScheduler implements ResourceScheduler { FSLeafQueue assignToQueue(RMApp rmApp, String queueName, String user) { FSLeafQueue queue = null; try { - QueuePlacementPolicy policy = queueMgr.getPlacementPolicy(); - queueName = policy.assignAppToQueue(queueName, user); + QueuePlacementPolicy placementPolicy = allocConf.getPlacementPolicy(); + queueName = placementPolicy.assignAppToQueue(queueName, user); if (queueName == null) { return null; } @@ -1128,27 +1132,27 @@ public class FairScheduler implements ResourceScheduler { @Override public synchronized void reinitialize(Configuration conf, RMContext rmContext) throws IOException { - this.conf = new FairSchedulerConfiguration(conf); - validateConf(this.conf); - minimumAllocation = this.conf.getMinimumAllocation(); - maximumAllocation = this.conf.getMaximumAllocation(); - incrAllocation = this.conf.getIncrementAllocation(); - continuousSchedulingEnabled = this.conf.isContinuousSchedulingEnabled(); - continuousSchedulingSleepMs = - this.conf.getContinuousSchedulingSleepMs(); - nodeLocalityThreshold = this.conf.getLocalityThresholdNode(); - rackLocalityThreshold = this.conf.getLocalityThresholdRack(); - nodeLocalityDelayMs = this.conf.getLocalityDelayNodeMs(); - rackLocalityDelayMs = this.conf.getLocalityDelayRackMs(); - preemptionEnabled = this.conf.getPreemptionEnabled(); - assignMultiple = this.conf.getAssignMultiple(); - maxAssign = this.conf.getMaxAssign(); - sizeBasedWeight = this.conf.getSizeBasedWeight(); - preemptionInterval = this.conf.getPreemptionInterval(); - waitTimeBeforeKill = this.conf.getWaitTimeBeforeKill(); - usePortForNodeName = this.conf.getUsePortForNodeName(); - if (!initialized) { + this.conf = new FairSchedulerConfiguration(conf); + validateConf(this.conf); + minimumAllocation = this.conf.getMinimumAllocation(); + maximumAllocation = this.conf.getMaximumAllocation(); + incrAllocation = this.conf.getIncrementAllocation(); + continuousSchedulingEnabled = this.conf.isContinuousSchedulingEnabled(); + continuousSchedulingSleepMs = + this.conf.getContinuousSchedulingSleepMs(); + nodeLocalityThreshold = this.conf.getLocalityThresholdNode(); + rackLocalityThreshold = this.conf.getLocalityThresholdRack(); + nodeLocalityDelayMs = this.conf.getLocalityDelayNodeMs(); + rackLocalityDelayMs = this.conf.getLocalityDelayRackMs(); + preemptionEnabled = this.conf.getPreemptionEnabled(); + assignMultiple = this.conf.getAssignMultiple(); + maxAssign = this.conf.getMaxAssign(); + sizeBasedWeight = this.conf.getSizeBasedWeight(); + preemptionInterval = this.conf.getPreemptionInterval(); + waitTimeBeforeKill = this.conf.getWaitTimeBeforeKill(); + usePortForNodeName = this.conf.getUsePortForNodeName(); + rootMetrics = FSQueueMetrics.forQueue("root", null, true, conf); this.rmContext = rmContext; this.eventLog = new FairSchedulerEventLog(); @@ -1156,8 +1160,9 @@ public class FairScheduler implements ResourceScheduler { initialized = true; + allocConf = new AllocationConfiguration(conf); try { - queueMgr.initialize(); + queueMgr.initialize(conf); } catch (Exception e) { throw new IOException("Failed to start FairScheduler", e); } @@ -1181,12 +1186,24 @@ public class FairScheduler implements ResourceScheduler { schedulingThread.setDaemon(true); schedulingThread.start(); } - } else { + + allocsLoader.init(conf); + allocsLoader.setReloadListener(new AllocationReloadListener()); + // If we fail to load allocations file on initialize, we want to fail + // immediately. After a successful load, exceptions on future reloads + // will just result in leaving things as they are. try { - queueMgr.reloadAllocs(); + allocsLoader.reloadAllocations(); } catch (Exception e) { throw new IOException("Failed to initialize FairScheduler", e); } + allocsLoader.start(); + } else { + try { + allocsLoader.reloadAllocations(); + } catch (Exception e) { + LOG.error("Failed to reload allocations file", e); + } } } @@ -1230,5 +1247,24 @@ public class FairScheduler implements ResourceScheduler { } return queue.hasAccess(acl, callerUGI); } + + public AllocationConfiguration getAllocationConfiguration() { + return allocConf; + } + + private class AllocationReloadListener implements + AllocationFileLoaderService.Listener { + + @Override + public void onReload(AllocationConfiguration queueInfo) { + // Commit the reload; also create any queue defined in the alloc file + // if it does not already exist, so it can be displayed on the web UI. + synchronized (FairScheduler.this) { + allocConf = queueInfo; + allocConf.getDefaultSchedulingPolicy().initialize(clusterCapacity); + queueMgr.updateAllocationConfiguration(allocConf); + } + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java index 955b102fee4..ec45cca158e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java @@ -18,7 +18,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import java.io.File; -import java.net.URL; +import java.util.ArrayList; +import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -151,14 +152,6 @@ public class FairSchedulerConfiguration extends Configuration { return Resources.createResource(incrementMemory, incrementCores); } - public boolean getAllowUndeclaredPools() { - return getBoolean(ALLOW_UNDECLARED_POOLS, DEFAULT_ALLOW_UNDECLARED_POOLS); - } - - public boolean getUserAsDefaultQueue() { - return getBoolean(USER_AS_DEFAULT_QUEUE, DEFAULT_USER_AS_DEFAULT_QUEUE); - } - public float getLocalityThresholdNode() { return getFloat(LOCALITY_THRESHOLD_NODE, DEFAULT_LOCALITY_THRESHOLD_NODE); } @@ -199,30 +192,6 @@ public class FairSchedulerConfiguration extends Configuration { return getBoolean(SIZE_BASED_WEIGHT, DEFAULT_SIZE_BASED_WEIGHT); } - /** - * Path to XML file containing allocations. If the - * path is relative, it is searched for in the - * classpath, but loaded like a regular File. - */ - public File getAllocationFile() { - String allocFilePath = get(ALLOCATION_FILE, DEFAULT_ALLOCATION_FILE); - File allocFile = new File(allocFilePath); - if (!allocFile.isAbsolute()) { - URL url = Thread.currentThread().getContextClassLoader() - .getResource(allocFilePath); - if (url == null) { - LOG.warn(allocFilePath + " not found on the classpath."); - allocFile = null; - } else if (!url.getProtocol().equalsIgnoreCase("file")) { - throw new RuntimeException("Allocation file " + url - + " found on the classpath is not on the local filesystem."); - } else { - allocFile = new File(url.getPath()); - } - } - return allocFile; - } - public String getEventlogDir() { return get(EVENT_LOG_DIR, new File(System.getProperty("hadoop.log.dir", "/tmp/")).getAbsolutePath() + File.separator + "fairscheduler"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java index e601086b8c4..35bca321f78 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java @@ -33,15 +33,15 @@ import com.google.common.collect.ListMultimap; * constraints */ public class MaxRunningAppsEnforcer { - private final QueueManager queueMgr; + private final FairScheduler scheduler; // Tracks the number of running applications by user. private final Map usersNumRunnableApps; @VisibleForTesting final ListMultimap usersNonRunnableApps; - public MaxRunningAppsEnforcer(QueueManager queueMgr) { - this.queueMgr = queueMgr; + public MaxRunningAppsEnforcer(FairScheduler scheduler) { + this.scheduler = scheduler; this.usersNumRunnableApps = new HashMap(); this.usersNonRunnableApps = ArrayListMultimap.create(); } @@ -51,16 +51,17 @@ public class MaxRunningAppsEnforcer { * maxRunningApps limits. */ public boolean canAppBeRunnable(FSQueue queue, String user) { + AllocationConfiguration allocConf = scheduler.getAllocationConfiguration(); Integer userNumRunnable = usersNumRunnableApps.get(user); if (userNumRunnable == null) { userNumRunnable = 0; } - if (userNumRunnable >= queueMgr.getUserMaxApps(user)) { + if (userNumRunnable >= allocConf.getUserMaxApps(user)) { return false; } // Check queue and all parent queues while (queue != null) { - int queueMaxApps = queueMgr.getQueueMaxApps(queue.getName()); + int queueMaxApps = allocConf.getQueueMaxApps(queue.getName()); if (queue.getNumRunnableApps() >= queueMaxApps) { return false; } @@ -107,6 +108,8 @@ public class MaxRunningAppsEnforcer { * highest queue that went from having no slack to having slack. */ public void updateRunnabilityOnAppRemoval(FSSchedulerApp app) { + AllocationConfiguration allocConf = scheduler.getAllocationConfiguration(); + // Update usersRunnableApps String user = app.getUser(); int newUserNumRunning = usersNumRunnableApps.get(user) - 1; @@ -127,10 +130,10 @@ public class MaxRunningAppsEnforcer { // that was at its maxRunningApps before the removal. FSLeafQueue queue = app.getQueue(); FSQueue highestQueueWithAppsNowRunnable = (queue.getNumRunnableApps() == - queueMgr.getQueueMaxApps(queue.getName()) - 1) ? queue : null; + allocConf.getQueueMaxApps(queue.getName()) - 1) ? queue : null; FSParentQueue parent = queue.getParent(); while (parent != null) { - if (parent.getNumRunnableApps() == queueMgr.getQueueMaxApps(parent + if (parent.getNumRunnableApps() == allocConf.getQueueMaxApps(parent .getName())) { highestQueueWithAppsNowRunnable = parent; } @@ -149,7 +152,7 @@ public class MaxRunningAppsEnforcer { gatherPossiblyRunnableAppLists(highestQueueWithAppsNowRunnable, appsNowMaybeRunnable); } - if (newUserNumRunning == queueMgr.getUserMaxApps(user) - 1) { + if (newUserNumRunning == allocConf.getUserMaxApps(user) - 1) { List userWaitingApps = usersNonRunnableApps.get(user); if (userWaitingApps != null) { appsNowMaybeRunnable.add(userWaitingApps); @@ -200,7 +203,8 @@ public class MaxRunningAppsEnforcer { */ private void gatherPossiblyRunnableAppLists(FSQueue queue, List> appLists) { - if (queue.getNumRunnableApps() < queueMgr.getQueueMaxApps(queue.getName())) { + if (queue.getNumRunnableApps() < scheduler.getAllocationConfiguration() + .getQueueMaxApps(queue.getName())) { if (queue instanceof FSLeafQueue) { appLists.add(((FSLeafQueue)queue).getNonRunnableAppSchedulables()); } else { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java index 95dfa4aff6e..38c338a399c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java @@ -18,20 +18,14 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; -import java.io.File; import java.io.IOException; -import java.net.URL; -import java.net.URLConnection; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import org.apache.commons.logging.Log; @@ -39,21 +33,9 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.authorize.AccessControlList; -import org.apache.hadoop.yarn.api.records.QueueACL; -import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; -import org.apache.hadoop.yarn.util.resource.Resources; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; -import org.w3c.dom.Text; import org.xml.sax.SAXException; -import com.google.common.annotations.VisibleForTesting; - /** * Maintains a list of queues as well as scheduling parameters for each queue, * such as guaranteed share allocations, from the fair scheduler config file. @@ -67,37 +49,13 @@ public class QueueManager { public static final String ROOT_QUEUE = "root"; - /** Time to wait between checks of the allocation file */ - public static final long ALLOC_RELOAD_INTERVAL = 10 * 1000; - - /** - * Time to wait after the allocation has been modified before reloading it - * (this is done to prevent loading a file that hasn't been fully written). - */ - public static final long ALLOC_RELOAD_WAIT = 5 * 1000; - - private static final AccessControlList EVERYBODY_ACL = new AccessControlList("*"); - private static final AccessControlList NOBODY_ACL = new AccessControlList(" "); - private final FairScheduler scheduler; - // Path to XML file containing allocations. - private File allocFile; - private final Collection leafQueues = new CopyOnWriteArrayList(); private final Map queues = new HashMap(); private FSParentQueue rootQueue; - @VisibleForTesting - volatile QueueManagerInfo info = new QueueManagerInfo(); - @VisibleForTesting - volatile QueuePlacementPolicy placementPolicy; - - private long lastReloadAttempt; // Last time we tried to reload the queues file - private long lastSuccessfulReload; // Last time we successfully reloaded queues - private boolean lastReloadAttemptFailed = false; - public QueueManager(FairScheduler scheduler) { this.scheduler = scheduler; } @@ -106,45 +64,15 @@ public class QueueManager { return rootQueue; } - public void initialize() throws IOException, SAXException, - AllocationConfigurationException, ParserConfigurationException { - FairSchedulerConfiguration conf = scheduler.getConf(); - rootQueue = new FSParentQueue("root", this, scheduler, null); + public void initialize(Configuration conf) throws IOException, + SAXException, AllocationConfigurationException, ParserConfigurationException { + rootQueue = new FSParentQueue("root", scheduler, null); queues.put(rootQueue.getName(), rootQueue); - this.allocFile = conf.getAllocationFile(); - placementPolicy = new QueuePlacementPolicy(getSimplePlacementRules(), - new HashSet(), conf); - - reloadAllocs(); - lastSuccessfulReload = scheduler.getClock().getTime(); - lastReloadAttempt = scheduler.getClock().getTime(); // Create the default queue getLeafQueue(YarnConfiguration.DEFAULT_QUEUE_NAME, true); } - public void updatePlacementPolicy(FairSchedulerConfiguration conf) { - - } - - /** - * Construct simple queue placement policy from allow-undeclared-pools and - * user-as-default-queue. - */ - private List getSimplePlacementRules() { - boolean create = scheduler.getConf().getAllowUndeclaredPools(); - boolean userAsDefaultQueue = scheduler.getConf().getUserAsDefaultQueue(); - List rules = new ArrayList(); - rules.add(new QueuePlacementRule.Specified().initialize(create, null)); - if (userAsDefaultQueue) { - rules.add(new QueuePlacementRule.User().initialize(create, null)); - } - if (!userAsDefaultQueue || !create) { - rules.add(new QueuePlacementRule.Default().initialize(true, null)); - } - return rules; - } - /** * Get a queue by name, creating it if the create param is true and is necessary. * If the queue is not or can not be a leaf queue, i.e. it already exists as a @@ -213,17 +141,30 @@ public class QueueManager { // queue to create. // Now that we know everything worked out, make all the queues // and add them to the map. + AllocationConfiguration queueConf = scheduler.getAllocationConfiguration(); FSLeafQueue leafQueue = null; for (int i = newQueueNames.size()-1; i >= 0; i--) { String queueName = newQueueNames.get(i); if (i == 0) { // First name added was the leaf queue - leafQueue = new FSLeafQueue(name, this, scheduler, parent); + leafQueue = new FSLeafQueue(name, scheduler, parent); + try { + leafQueue.setPolicy(queueConf.getDefaultSchedulingPolicy()); + } catch (AllocationConfigurationException ex) { + LOG.warn("Failed to set default scheduling policy " + + queueConf.getDefaultSchedulingPolicy() + " on new leaf queue.", ex); + } parent.addChildQueue(leafQueue); queues.put(leafQueue.getName(), leafQueue); leafQueues.add(leafQueue); } else { - FSParentQueue newParent = new FSParentQueue(queueName, this, scheduler, parent); + FSParentQueue newParent = new FSParentQueue(queueName, scheduler, parent); + try { + newParent.setPolicy(queueConf.getDefaultSchedulingPolicy()); + } catch (AllocationConfigurationException ex) { + LOG.warn("Failed to set default scheduling policy " + + queueConf.getDefaultSchedulingPolicy() + " on new parent queue.", ex); + } parent.addChildQueue(newParent); queues.put(newParent.getName(), newParent); parent = newParent; @@ -257,301 +198,6 @@ public class QueueManager { } } - public QueuePlacementPolicy getPlacementPolicy() { - return placementPolicy; - } - - /** - * Reload allocations file if it hasn't been loaded in a while - */ - public void reloadAllocsIfNecessary() { - long time = scheduler.getClock().getTime(); - if (time > lastReloadAttempt + ALLOC_RELOAD_INTERVAL) { - lastReloadAttempt = time; - if (null == allocFile) { - return; - } - try { - // Get last modified time of alloc file depending whether it's a String - // (for a path name) or an URL (for a classloader resource) - long lastModified = allocFile.lastModified(); - if (lastModified > lastSuccessfulReload && - time > lastModified + ALLOC_RELOAD_WAIT) { - reloadAllocs(); - lastSuccessfulReload = time; - lastReloadAttemptFailed = false; - } - } catch (Exception e) { - // Throwing the error further out here won't help - the RPC thread - // will catch it and report it in a loop. Instead, just log it and - // hope somebody will notice from the log. - // We log the error only on the first failure so we don't fill up the - // JobTracker's log with these messages. - if (!lastReloadAttemptFailed) { - LOG.error("Failed to reload fair scheduler config file - " + - "will use existing allocations.", e); - } - lastReloadAttemptFailed = true; - } - } - } - - /** - * Updates the allocation list from the allocation config file. This file is - * expected to be in the XML format specified in the design doc. - * - * @throws IOException if the config file cannot be read. - * @throws AllocationConfigurationException if allocations are invalid. - * @throws ParserConfigurationException if XML parser is misconfigured. - * @throws SAXException if config file is malformed. - */ - public void reloadAllocs() throws IOException, ParserConfigurationException, - SAXException, AllocationConfigurationException { - if (allocFile == null) return; - // Create some temporary hashmaps to hold the new allocs, and we only save - // them in our fields if we have parsed the entire allocs file successfully. - Map minQueueResources = new HashMap(); - Map maxQueueResources = new HashMap(); - Map queueMaxApps = new HashMap(); - Map userMaxApps = new HashMap(); - Map queueWeights = new HashMap(); - Map queuePolicies = new HashMap(); - Map minSharePreemptionTimeouts = new HashMap(); - Map> queueAcls = - new HashMap>(); - int userMaxAppsDefault = Integer.MAX_VALUE; - int queueMaxAppsDefault = Integer.MAX_VALUE; - long fairSharePreemptionTimeout = Long.MAX_VALUE; - long defaultMinSharePreemptionTimeout = Long.MAX_VALUE; - SchedulingPolicy defaultSchedPolicy = SchedulingPolicy.getDefault(); - - QueuePlacementPolicy newPlacementPolicy = null; - - // Remember all queue names so we can display them on web UI, etc. - List queueNamesInAllocFile = new ArrayList(); - - // Read and parse the allocations file. - DocumentBuilderFactory docBuilderFactory = - DocumentBuilderFactory.newInstance(); - docBuilderFactory.setIgnoringComments(true); - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(allocFile); - Element root = doc.getDocumentElement(); - if (!"allocations".equals(root.getTagName())) - throw new AllocationConfigurationException("Bad fair scheduler config " + - "file: top-level element not "); - NodeList elements = root.getChildNodes(); - List queueElements = new ArrayList(); - Element placementPolicyElement = null; - for (int i = 0; i < elements.getLength(); i++) { - Node node = elements.item(i); - if (node instanceof Element) { - Element element = (Element)node; - if ("queue".equals(element.getTagName()) || - "pool".equals(element.getTagName())) { - queueElements.add(element); - } else if ("user".equals(element.getTagName())) { - String userName = element.getAttribute("name"); - NodeList fields = element.getChildNodes(); - for (int j = 0; j < fields.getLength(); j++) { - Node fieldNode = fields.item(j); - if (!(fieldNode instanceof Element)) - continue; - Element field = (Element) fieldNode; - if ("maxRunningApps".equals(field.getTagName())) { - String text = ((Text)field.getFirstChild()).getData().trim(); - int val = Integer.parseInt(text); - userMaxApps.put(userName, val); - } - } - } else if ("userMaxAppsDefault".equals(element.getTagName())) { - String text = ((Text)element.getFirstChild()).getData().trim(); - int val = Integer.parseInt(text); - userMaxAppsDefault = val; - } else if ("fairSharePreemptionTimeout".equals(element.getTagName())) { - String text = ((Text)element.getFirstChild()).getData().trim(); - long val = Long.parseLong(text) * 1000L; - fairSharePreemptionTimeout = val; - } else if ("defaultMinSharePreemptionTimeout".equals(element.getTagName())) { - String text = ((Text)element.getFirstChild()).getData().trim(); - long val = Long.parseLong(text) * 1000L; - defaultMinSharePreemptionTimeout = val; - } else if ("queueMaxAppsDefault".equals(element.getTagName())) { - String text = ((Text)element.getFirstChild()).getData().trim(); - int val = Integer.parseInt(text); - queueMaxAppsDefault = val; - } else if ("defaultQueueSchedulingPolicy".equals(element.getTagName()) - || "defaultQueueSchedulingMode".equals(element.getTagName())) { - String text = ((Text)element.getFirstChild()).getData().trim(); - SchedulingPolicy.setDefault(text); - defaultSchedPolicy = SchedulingPolicy.getDefault(); - } else if ("queuePlacementPolicy".equals(element.getTagName())) { - placementPolicyElement = element; - } else { - LOG.warn("Bad element in allocations file: " + element.getTagName()); - } - } - } - - // Load queue elements. A root queue can either be included or omitted. If - // it's included, all other queues must be inside it. - for (Element element : queueElements) { - String parent = "root"; - if (element.getAttribute("name").equalsIgnoreCase("root")) { - if (queueElements.size() > 1) { - throw new AllocationConfigurationException("If configuring root queue," - + " no other queues can be placed alongside it."); - } - parent = null; - } - loadQueue(parent, element, minQueueResources, maxQueueResources, queueMaxApps, - userMaxApps, queueWeights, queuePolicies, minSharePreemptionTimeouts, - queueAcls, queueNamesInAllocFile); - } - - // Load placement policy and pass it configured queues - if (placementPolicyElement != null) { - newPlacementPolicy = QueuePlacementPolicy.fromXml(placementPolicyElement, - new HashSet(queueNamesInAllocFile), scheduler.getConf()); - } else { - newPlacementPolicy = new QueuePlacementPolicy(getSimplePlacementRules(), - new HashSet(queueNamesInAllocFile), scheduler.getConf()); - } - - // Commit the reload; also create any queue defined in the alloc file - // if it does not already exist, so it can be displayed on the web UI. - synchronized (this) { - info = new QueueManagerInfo(minQueueResources, maxQueueResources, - queueMaxApps, userMaxApps, queueWeights, userMaxAppsDefault, - queueMaxAppsDefault, defaultSchedPolicy, minSharePreemptionTimeouts, - queueAcls, fairSharePreemptionTimeout, defaultMinSharePreemptionTimeout); - placementPolicy = newPlacementPolicy; - - // Make sure all queues exist - for (String name: queueNamesInAllocFile) { - getLeafQueue(name, true); - } - - for (FSQueue queue : queues.values()) { - // Update queue metrics - FSQueueMetrics queueMetrics = queue.getMetrics(); - queueMetrics.setMinShare(queue.getMinShare()); - queueMetrics.setMaxShare(queue.getMaxShare()); - // Set scheduling policies - if (queuePolicies.containsKey(queue.getName())) { - queue.setPolicy(queuePolicies.get(queue.getName())); - } else { - queue.setPolicy(SchedulingPolicy.getDefault()); - } - } - - } - } - - /** - * Loads a queue from a queue element in the configuration file - */ - private void loadQueue(String parentName, Element element, Map minQueueResources, - Map maxQueueResources, Map queueMaxApps, - Map userMaxApps, Map queueWeights, - Map queuePolicies, - Map minSharePreemptionTimeouts, - Map> queueAcls, List queueNamesInAllocFile) - throws AllocationConfigurationException { - String queueName = element.getAttribute("name"); - if (parentName != null) { - queueName = parentName + "." + queueName; - } - Map acls = - new HashMap(); - NodeList fields = element.getChildNodes(); - boolean isLeaf = true; - - for (int j = 0; j < fields.getLength(); j++) { - Node fieldNode = fields.item(j); - if (!(fieldNode instanceof Element)) - continue; - Element field = (Element) fieldNode; - if ("minResources".equals(field.getTagName())) { - String text = ((Text)field.getFirstChild()).getData().trim(); - Resource val = FairSchedulerConfiguration.parseResourceConfigValue(text); - minQueueResources.put(queueName, val); - } else if ("maxResources".equals(field.getTagName())) { - String text = ((Text)field.getFirstChild()).getData().trim(); - Resource val = FairSchedulerConfiguration.parseResourceConfigValue(text); - maxQueueResources.put(queueName, val); - } else if ("maxRunningApps".equals(field.getTagName())) { - String text = ((Text)field.getFirstChild()).getData().trim(); - int val = Integer.parseInt(text); - queueMaxApps.put(queueName, val); - } else if ("weight".equals(field.getTagName())) { - String text = ((Text)field.getFirstChild()).getData().trim(); - double val = Double.parseDouble(text); - queueWeights.put(queueName, new ResourceWeights((float)val)); - } else if ("minSharePreemptionTimeout".equals(field.getTagName())) { - String text = ((Text)field.getFirstChild()).getData().trim(); - long val = Long.parseLong(text) * 1000L; - minSharePreemptionTimeouts.put(queueName, val); - } else if ("schedulingPolicy".equals(field.getTagName()) - || "schedulingMode".equals(field.getTagName())) { - String text = ((Text)field.getFirstChild()).getData().trim(); - SchedulingPolicy policy = SchedulingPolicy.parse(text); - policy.initialize(scheduler.getClusterCapacity()); - queuePolicies.put(queueName, policy); - } else if ("aclSubmitApps".equals(field.getTagName())) { - String text = ((Text)field.getFirstChild()).getData(); - acls.put(QueueACL.SUBMIT_APPLICATIONS, new AccessControlList(text)); - } else if ("aclAdministerApps".equals(field.getTagName())) { - String text = ((Text)field.getFirstChild()).getData(); - acls.put(QueueACL.ADMINISTER_QUEUE, new AccessControlList(text)); - } else if ("queue".endsWith(field.getTagName()) || - "pool".equals(field.getTagName())) { - loadQueue(queueName, field, minQueueResources, maxQueueResources, - queueMaxApps, userMaxApps, queueWeights, queuePolicies, - minSharePreemptionTimeouts, - queueAcls, queueNamesInAllocFile); - isLeaf = false; - } - } - if (isLeaf) { - queueNamesInAllocFile.add(queueName); - } - queueAcls.put(queueName, acls); - if (maxQueueResources.containsKey(queueName) && minQueueResources.containsKey(queueName) - && !Resources.fitsIn(minQueueResources.get(queueName), - maxQueueResources.get(queueName))) { - LOG.warn(String.format("Queue %s has max resources %d less than min resources %d", - queueName, maxQueueResources.get(queueName), minQueueResources.get(queueName))); - } - } - - /** - * Get the minimum resource allocation for the given queue. - * @return the cap set on this queue, or 0 if not set. - */ - public Resource getMinResources(String queue) { - Resource minQueueResource = info.minQueueResources.get(queue); - if (minQueueResource != null) { - return minQueueResource; - } else { - return Resources.createResource(0); - } - } - - /** - * Get the maximum resource allocation for the given queue. - * @return the cap set on this queue, or Integer.MAX_VALUE if not set. - */ - - public Resource getMaxResources(String queueName) { - Resource maxQueueResource = info.maxQueueResources.get(queueName); - if (maxQueueResource != null) { - return maxQueueResource; - } else { - return Resources.createResource(Integer.MAX_VALUE, Integer.MAX_VALUE); - } - } - /** * Get a collection of all leaf queues */ @@ -567,141 +213,27 @@ public class QueueManager { public Collection getQueues() { return queues.values(); } - - public int getUserMaxApps(String user) { - // save current info in case it gets changed under us - QueueManagerInfo info = this.info; - if (info.userMaxApps.containsKey(user)) { - return info.userMaxApps.get(user); - } else { - return info.userMaxAppsDefault; - } - } - - public int getQueueMaxApps(String queue) { - // save current info in case it gets changed under us - QueueManagerInfo info = this.info; - if (info.queueMaxApps.containsKey(queue)) { - return info.queueMaxApps.get(queue); - } else { - return info.queueMaxAppsDefault; - } - } - public ResourceWeights getQueueWeight(String queue) { - ResourceWeights weight = info.queueWeights.get(queue); - if (weight != null) { - return weight; - } else { - return ResourceWeights.NEUTRAL; - } - } - - /** - * Get a queue's min share preemption timeout, in milliseconds. This is the - * time after which jobs in the queue may kill other queues' tasks if they - * are below their min share. - */ - public long getMinSharePreemptionTimeout(String queueName) { - // save current info in case it gets changed under us - QueueManagerInfo info = this.info; - if (info.minSharePreemptionTimeouts.containsKey(queueName)) { - return info.minSharePreemptionTimeouts.get(queueName); - } - return info.defaultMinSharePreemptionTimeout; - } - - /** - * Get the fair share preemption, in milliseconds. This is the time - * after which any job may kill other jobs' tasks if it is below half - * its fair share. - */ - public long getFairSharePreemptionTimeout() { - return info.fairSharePreemptionTimeout; - } - - /** - * Get the ACLs associated with this queue. If a given ACL is not explicitly - * configured, include the default value for that ACL. The default for the - * root queue is everybody ("*") and the default for all other queues is - * nobody ("") - */ - public AccessControlList getQueueAcl(String queue, QueueACL operation) { - Map queueAcls = info.queueAcls.get(queue); - if (queueAcls == null || !queueAcls.containsKey(operation)) { - return (queue.equals(ROOT_QUEUE)) ? EVERYBODY_ACL : NOBODY_ACL; - } - return queueAcls.get(operation); - } - - static class QueueManagerInfo { - // Minimum resource allocation for each queue - public final Map minQueueResources; - // Maximum amount of resources per queue - public final Map maxQueueResources; - // Sharing weights for each queue - public final Map queueWeights; - - // Max concurrent running applications for each queue and for each user; in addition, - // for users that have no max specified, we use the userMaxJobsDefault. - public final Map queueMaxApps; - public final Map userMaxApps; - public final int userMaxAppsDefault; - public final int queueMaxAppsDefault; - - // ACL's for each queue. Only specifies non-default ACL's from configuration. - public final Map> queueAcls; - - // Min share preemption timeout for each queue in seconds. If a job in the queue - // waits this long without receiving its guaranteed share, it is allowed to - // preempt other jobs' tasks. - public final Map minSharePreemptionTimeouts; - - // Default min share preemption timeout for queues where it is not set - // explicitly. - public final long defaultMinSharePreemptionTimeout; - - // Preemption timeout for jobs below fair share in seconds. If a job remains - // below half its fair share for this long, it is allowed to preempt tasks. - public final long fairSharePreemptionTimeout; - - public final SchedulingPolicy defaultSchedulingPolicy; - - public QueueManagerInfo(Map minQueueResources, - Map maxQueueResources, - Map queueMaxApps, Map userMaxApps, - Map queueWeights, int userMaxAppsDefault, - int queueMaxAppsDefault, SchedulingPolicy defaultSchedulingPolicy, - Map minSharePreemptionTimeouts, - Map> queueAcls, - long fairSharePreemptionTimeout, long defaultMinSharePreemptionTimeout) { - this.minQueueResources = minQueueResources; - this.maxQueueResources = maxQueueResources; - this.queueMaxApps = queueMaxApps; - this.userMaxApps = userMaxApps; - this.queueWeights = queueWeights; - this.userMaxAppsDefault = userMaxAppsDefault; - this.queueMaxAppsDefault = queueMaxAppsDefault; - this.defaultSchedulingPolicy = defaultSchedulingPolicy; - this.minSharePreemptionTimeouts = minSharePreemptionTimeouts; - this.queueAcls = queueAcls; - this.fairSharePreemptionTimeout = fairSharePreemptionTimeout; - this.defaultMinSharePreemptionTimeout = defaultMinSharePreemptionTimeout; + public void updateAllocationConfiguration(AllocationConfiguration queueConf) { + // Make sure all queues exist + for (String name : queueConf.getQueueNames()) { + getLeafQueue(name, true); } - public QueueManagerInfo() { - minQueueResources = new HashMap(); - maxQueueResources = new HashMap(); - queueWeights = new HashMap(); - queueMaxApps = new HashMap(); - userMaxApps = new HashMap(); - userMaxAppsDefault = Integer.MAX_VALUE; - queueMaxAppsDefault = Integer.MAX_VALUE; - queueAcls = new HashMap>(); - minSharePreemptionTimeouts = new HashMap(); - defaultMinSharePreemptionTimeout = Long.MAX_VALUE; - fairSharePreemptionTimeout = Long.MAX_VALUE; - defaultSchedulingPolicy = SchedulingPolicy.getDefault(); + for (FSQueue queue : queues.values()) { + // Update queue metrics + FSQueueMetrics queueMetrics = queue.getMetrics(); + queueMetrics.setMinShare(queue.getMinShare()); + queueMetrics.setMaxShare(queue.getMaxShare()); + // Set scheduling policies + try { + SchedulingPolicy policy = queueConf.getSchedulingPolicy(queue.getName()); + policy.initialize(scheduler.getClusterCapacity()); + queue.setPolicy(policy); + } catch (AllocationConfigurationException ex) { + LOG.warn("Cannot apply configured scheduling policy to queue " + + queue.getName(), ex); + } } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java index 4bf6b613166..d802e709644 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java @@ -94,6 +94,34 @@ public class QueuePlacementPolicy { return new QueuePlacementPolicy(rules, configuredQueues, conf); } + /** + * Build a simple queue placement policy from the allow-undeclared-pools and + * user-as-default-queue configuration options. + */ + public static QueuePlacementPolicy fromConfiguration(Configuration conf, + Set configuredQueues) { + boolean create = conf.getBoolean( + FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, + FairSchedulerConfiguration.DEFAULT_ALLOW_UNDECLARED_POOLS); + boolean userAsDefaultQueue = conf.getBoolean( + FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, + FairSchedulerConfiguration.DEFAULT_USER_AS_DEFAULT_QUEUE); + List rules = new ArrayList(); + rules.add(new QueuePlacementRule.Specified().initialize(create, null)); + if (userAsDefaultQueue) { + rules.add(new QueuePlacementRule.User().initialize(create, null)); + } + if (!userAsDefaultQueue || !create) { + rules.add(new QueuePlacementRule.Default().initialize(true, null)); + } + try { + return new QueuePlacementPolicy(rules, configuredQueues, conf); + } catch (AllocationConfigurationException ex) { + throw new RuntimeException("Should never hit exception when loading" + + "placement policy from conf", ex); + } + } + /** * Applies this rule to an app with the given requested queue and user/group * information. @@ -120,4 +148,8 @@ public class QueuePlacementPolicy { throw new IllegalStateException("Should have applied a rule before " + "reaching here"); } + + public List getRules() { + return rules; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java index 06f384045e0..549b85c380f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java @@ -35,7 +35,7 @@ public abstract class SchedulingPolicy { private static final ConcurrentHashMap, SchedulingPolicy> instances = new ConcurrentHashMap, SchedulingPolicy>(); - private static SchedulingPolicy DEFAULT_POLICY = + public static final SchedulingPolicy DEFAULT_POLICY = getInstance(FairSharePolicy.class); public static final byte DEPTH_LEAF = (byte) 1; @@ -44,15 +44,6 @@ public abstract class SchedulingPolicy { public static final byte DEPTH_PARENT = (byte) 6; // Root and Intermediate public static final byte DEPTH_ANY = (byte) 7; - public static SchedulingPolicy getDefault() { - return DEFAULT_POLICY; - } - - public static void setDefault(String className) - throws AllocationConfigurationException { - DEFAULT_POLICY = parse(className); - } - /** * Returns a {@link SchedulingPolicy} instance corresponding to the passed clazz */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java index fc4732d6bfe..1c5a79bed91 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java @@ -29,10 +29,10 @@ import javax.xml.bind.annotation.XmlSeeAlso; import javax.xml.bind.annotation.XmlTransient; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager; import org.apache.hadoop.yarn.util.resource.Resources; @XmlRootElement @@ -65,7 +65,7 @@ public class FairSchedulerQueueInfo { } public FairSchedulerQueueInfo(FSQueue queue, FairScheduler scheduler) { - QueueManager manager = scheduler.getQueueManager(); + AllocationConfiguration allocConf = scheduler.getAllocationConfiguration(); queueName = queue.getName(); schedulingPolicy = queue.getPolicy().getName(); @@ -87,7 +87,7 @@ public class FairSchedulerQueueInfo { fractionMemMinShare = (float)minResources.getMemory() / clusterResources.getMemory(); fractionMemMaxShare = (float)maxResources.getMemory() / clusterResources.getMemory(); - maxApps = manager.getQueueMaxApps(queueName); + maxApps = allocConf.getQueueMaxApps(queueName); Collection children = queue.getChildQueues(); childQueues = new ArrayList(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java new file mode 100644 index 00000000000..dc28a3b3ff4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java @@ -0,0 +1,432 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; + +import static junit.framework.Assert.*; +import static org.junit.Assert.assertEquals; + +import java.io.File; +import java.io.FileWriter; +import java.io.PrintWriter; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy; +import org.apache.hadoop.yarn.util.Clock; +import org.apache.hadoop.yarn.util.resource.Resources; +import org.junit.Test; + +public class TestAllocationFileLoaderService { + + final static String TEST_DIR = new File(System.getProperty("test.build.data", + "/tmp")).getAbsolutePath(); + + final static String ALLOC_FILE = new File(TEST_DIR, + "test-queues").getAbsolutePath(); + + private class MockClock implements Clock { + private long time = 0; + @Override + public long getTime() { + return time; + } + + public void tick(long ms) { + time += ms; + } + } + + @Test + public void testGetAllocationFileFromClasspath() { + Configuration conf = new Configuration(); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, + "test-fair-scheduler.xml"); + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + File allocationFile = allocLoader.getAllocationFile(conf); + assertEquals("test-fair-scheduler.xml", allocationFile.getName()); + assertTrue(allocationFile.exists()); + } + + @Test (timeout = 10000) + public void testReload() throws Exception { + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(" "); + out.println(" 1"); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(""); + out.close(); + + MockClock clock = new MockClock(); + Configuration conf = new Configuration(); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService( + clock); + allocLoader.reloadIntervalMs = 5; + allocLoader.init(conf); + ReloadListener confHolder = new ReloadListener(); + allocLoader.setReloadListener(confHolder); + allocLoader.reloadAllocations(); + AllocationConfiguration allocConf = confHolder.allocConf; + + // Verify conf + QueuePlacementPolicy policy = allocConf.getPlacementPolicy(); + List rules = policy.getRules(); + assertEquals(1, rules.size()); + assertEquals(QueuePlacementRule.Default.class, rules.get(0).getClass()); + assertEquals(1, allocConf.getQueueMaxApps("root.queueA")); + assertEquals(2, allocConf.getQueueNames().size()); + assertTrue(allocConf.getQueueNames().contains("root.queueA")); + assertTrue(allocConf.getQueueNames().contains("root.queueB")); + + confHolder.allocConf = null; + + // Modify file and advance the clock + out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(" "); + out.println(" 3"); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(""); + out.close(); + + clock.tick(System.currentTimeMillis() + + AllocationFileLoaderService.ALLOC_RELOAD_WAIT_MS + 10000); + allocLoader.start(); + + while (confHolder.allocConf == null) { + Thread.sleep(20); + } + + // Verify conf + allocConf = confHolder.allocConf; + policy = allocConf.getPlacementPolicy(); + rules = policy.getRules(); + assertEquals(2, rules.size()); + assertEquals(QueuePlacementRule.Specified.class, rules.get(0).getClass()); + assertEquals(QueuePlacementRule.Default.class, rules.get(1).getClass()); + assertEquals(3, allocConf.getQueueMaxApps("root.queueB")); + assertEquals(1, allocConf.getQueueNames().size()); + assertTrue(allocConf.getQueueNames().contains("root.queueB")); + } + + @Test + public void testAllocationFileParsing() throws Exception { + Configuration conf = new Configuration(); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + // Give queue A a minimum of 1024 M + out.println(""); + out.println("1024mb,0vcores"); + out.println(""); + // Give queue B a minimum of 2048 M + out.println(""); + out.println("2048mb,0vcores"); + out.println("alice,bob admins"); + out.println("fair"); + out.println(""); + // Give queue C no minimum + out.println(""); + out.println("alice,bob admins"); + out.println(""); + // Give queue D a limit of 3 running apps + out.println(""); + out.println("3"); + out.println(""); + // Give queue E a preemption timeout of one minute + out.println(""); + out.println("60"); + out.println(""); + // Set default limit of apps per queue to 15 + out.println("15"); + // Set default limit of apps per user to 5 + out.println("5"); + // Give user1 a limit of 10 jobs + out.println(""); + out.println("10"); + out.println(""); + // Set default min share preemption timeout to 2 minutes + out.println("120" + + ""); + // Set fair share preemption timeout to 5 minutes + out.println("300"); + // Set default scheduling policy to DRF + out.println("drf"); + out.println(""); + out.close(); + + allocLoader.init(conf); + ReloadListener confHolder = new ReloadListener(); + allocLoader.setReloadListener(confHolder); + allocLoader.reloadAllocations(); + AllocationConfiguration queueConf = confHolder.allocConf; + + assertEquals(5, queueConf.getQueueNames().size()); + assertEquals(Resources.createResource(0), + queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); + assertEquals(Resources.createResource(0), + queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); + + assertEquals(Resources.createResource(1024, 0), + queueConf.getMinResources("root.queueA")); + assertEquals(Resources.createResource(2048, 0), + queueConf.getMinResources("root.queueB")); + assertEquals(Resources.createResource(0), + queueConf.getMinResources("root.queueC")); + assertEquals(Resources.createResource(0), + queueConf.getMinResources("root.queueD")); + assertEquals(Resources.createResource(0), + queueConf.getMinResources("root.queueE")); + + assertEquals(15, queueConf.getQueueMaxApps("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); + assertEquals(15, queueConf.getQueueMaxApps("root.queueA")); + assertEquals(15, queueConf.getQueueMaxApps("root.queueB")); + assertEquals(15, queueConf.getQueueMaxApps("root.queueC")); + assertEquals(3, queueConf.getQueueMaxApps("root.queueD")); + assertEquals(15, queueConf.getQueueMaxApps("root.queueE")); + assertEquals(10, queueConf.getUserMaxApps("user1")); + assertEquals(5, queueConf.getUserMaxApps("user2")); + + // Root should get * ACL + assertEquals("*", queueConf.getQueueAcl("root", + QueueACL.ADMINISTER_QUEUE).getAclString()); + assertEquals("*", queueConf.getQueueAcl("root", + QueueACL.SUBMIT_APPLICATIONS).getAclString()); + + // Unspecified queues should get default ACL + assertEquals(" ", queueConf.getQueueAcl("root.queueA", + QueueACL.ADMINISTER_QUEUE).getAclString()); + assertEquals(" ", queueConf.getQueueAcl("root.queueA", + QueueACL.SUBMIT_APPLICATIONS).getAclString()); + + // Queue B ACL + assertEquals("alice,bob admins", queueConf.getQueueAcl("root.queueB", + QueueACL.ADMINISTER_QUEUE).getAclString()); + + // Queue C ACL + assertEquals("alice,bob admins", queueConf.getQueueAcl("root.queueC", + QueueACL.SUBMIT_APPLICATIONS).getAclString()); + + assertEquals(120000, queueConf.getMinSharePreemptionTimeout("root." + + YarnConfiguration.DEFAULT_QUEUE_NAME)); + assertEquals(120000, queueConf.getMinSharePreemptionTimeout("root.queueA")); + assertEquals(120000, queueConf.getMinSharePreemptionTimeout("root.queueB")); + assertEquals(120000, queueConf.getMinSharePreemptionTimeout("root.queueC")); + assertEquals(120000, queueConf.getMinSharePreemptionTimeout("root.queueD")); + assertEquals(120000, queueConf.getMinSharePreemptionTimeout("root.queueA")); + assertEquals(60000, queueConf.getMinSharePreemptionTimeout("root.queueE")); + assertEquals(300000, queueConf.getFairSharePreemptionTimeout()); + + // Verify existing queues have default scheduling policy + assertEquals(DominantResourceFairnessPolicy.NAME, + queueConf.getSchedulingPolicy("root").getName()); + assertEquals(DominantResourceFairnessPolicy.NAME, + queueConf.getSchedulingPolicy("root.queueA").getName()); + // Verify default is overriden if specified explicitly + assertEquals(FairSharePolicy.NAME, + queueConf.getSchedulingPolicy("root.queueB").getName()); + // Verify new queue gets default scheduling policy + assertEquals(DominantResourceFairnessPolicy.NAME, + queueConf.getSchedulingPolicy("root.newqueue").getName()); + } + + @Test + public void testBackwardsCompatibleAllocationFileParsing() throws Exception { + Configuration conf = new Configuration(); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + // Give queue A a minimum of 1024 M + out.println(""); + out.println("1024mb,0vcores"); + out.println(""); + // Give queue B a minimum of 2048 M + out.println(""); + out.println("2048mb,0vcores"); + out.println("alice,bob admins"); + out.println(""); + // Give queue C no minimum + out.println(""); + out.println("alice,bob admins"); + out.println(""); + // Give queue D a limit of 3 running apps + out.println(""); + out.println("3"); + out.println(""); + // Give queue E a preemption timeout of one minute + out.println(""); + out.println("60"); + out.println(""); + // Set default limit of apps per queue to 15 + out.println("15"); + // Set default limit of apps per user to 5 + out.println("5"); + // Give user1 a limit of 10 jobs + out.println(""); + out.println("10"); + out.println(""); + // Set default min share preemption timeout to 2 minutes + out.println("120" + + ""); + // Set fair share preemption timeout to 5 minutes + out.println("300"); + out.println(""); + out.close(); + + allocLoader.init(conf); + ReloadListener confHolder = new ReloadListener(); + allocLoader.setReloadListener(confHolder); + allocLoader.reloadAllocations(); + AllocationConfiguration queueConf = confHolder.allocConf; + + assertEquals(5, queueConf.getQueueNames().size()); + assertEquals(Resources.createResource(0), + queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); + assertEquals(Resources.createResource(0), + queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); + + assertEquals(Resources.createResource(1024, 0), + queueConf.getMinResources("root.queueA")); + assertEquals(Resources.createResource(2048, 0), + queueConf.getMinResources("root.queueB")); + assertEquals(Resources.createResource(0), + queueConf.getMinResources("root.queueC")); + assertEquals(Resources.createResource(0), + queueConf.getMinResources("root.queueD")); + assertEquals(Resources.createResource(0), + queueConf.getMinResources("root.queueE")); + + assertEquals(15, queueConf.getQueueMaxApps("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); + assertEquals(15, queueConf.getQueueMaxApps("root.queueA")); + assertEquals(15, queueConf.getQueueMaxApps("root.queueB")); + assertEquals(15, queueConf.getQueueMaxApps("root.queueC")); + assertEquals(3, queueConf.getQueueMaxApps("root.queueD")); + assertEquals(15, queueConf.getQueueMaxApps("root.queueE")); + assertEquals(10, queueConf.getUserMaxApps("user1")); + assertEquals(5, queueConf.getUserMaxApps("user2")); + + // Unspecified queues should get default ACL + assertEquals(" ", queueConf.getQueueAcl("root.queueA", + QueueACL.ADMINISTER_QUEUE).getAclString()); + assertEquals(" ", queueConf.getQueueAcl("root.queueA", + QueueACL.SUBMIT_APPLICATIONS).getAclString()); + + // Queue B ACL + assertEquals("alice,bob admins", queueConf.getQueueAcl("root.queueB", + QueueACL.ADMINISTER_QUEUE).getAclString()); + + // Queue C ACL + assertEquals("alice,bob admins", queueConf.getQueueAcl("root.queueC", + QueueACL.SUBMIT_APPLICATIONS).getAclString()); + + + assertEquals(120000, queueConf.getMinSharePreemptionTimeout("root." + + YarnConfiguration.DEFAULT_QUEUE_NAME)); + assertEquals(120000, queueConf.getMinSharePreemptionTimeout("root.queueA")); + assertEquals(120000, queueConf.getMinSharePreemptionTimeout("root.queueB")); + assertEquals(120000, queueConf.getMinSharePreemptionTimeout("root.queueC")); + assertEquals(120000, queueConf.getMinSharePreemptionTimeout("root.queueD")); + assertEquals(120000, queueConf.getMinSharePreemptionTimeout("root.queueA")); + assertEquals(60000, queueConf.getMinSharePreemptionTimeout("root.queueE")); + assertEquals(300000, queueConf.getFairSharePreemptionTimeout()); + } + + @Test + public void testSimplePlacementPolicyFromConf() throws Exception { + Configuration conf = new Configuration(); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false); + conf.setBoolean(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, false); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.close(); + + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + allocLoader.init(conf); + ReloadListener confHolder = new ReloadListener(); + allocLoader.setReloadListener(confHolder); + allocLoader.reloadAllocations(); + AllocationConfiguration allocConf = confHolder.allocConf; + + QueuePlacementPolicy placementPolicy = allocConf.getPlacementPolicy(); + List rules = placementPolicy.getRules(); + assertEquals(2, rules.size()); + assertEquals(QueuePlacementRule.Specified.class, rules.get(0).getClass()); + assertEquals(false, rules.get(0).create); + assertEquals(QueuePlacementRule.Default.class, rules.get(1).getClass()); + } + + /** + * Verify that you can't place queues at the same level as the root queue in + * the allocations file. + */ + @Test (expected = AllocationConfigurationException.class) + public void testQueueAlongsideRoot() throws Exception { + Configuration conf = new Configuration(); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.close(); + + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + allocLoader.init(conf); + ReloadListener confHolder = new ReloadListener(); + allocLoader.setReloadListener(confHolder); + allocLoader.reloadAllocations(); + } + + private class ReloadListener implements AllocationFileLoaderService.Listener { + public AllocationConfiguration allocConf; + + @Override + public void onReload(AllocationConfiguration info) { + allocConf = info; + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java index 5bfb182ced3..e4dc8016c12 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java @@ -51,11 +51,11 @@ public class TestFSLeafQueue { scheduler.reinitialize(conf, resourceManager.getRMContext()); String queueName = "root.queue1"; - QueueManager mockMgr = mock(QueueManager.class); - when(mockMgr.getMaxResources(queueName)).thenReturn(maxResource); - when(mockMgr.getMinResources(queueName)).thenReturn(Resources.none()); + scheduler.allocConf = mock(AllocationConfiguration.class); + when(scheduler.allocConf.getMaxResources(queueName)).thenReturn(maxResource); + when(scheduler.allocConf.getMinResources(queueName)).thenReturn(Resources.none()); - schedulable = new FSLeafQueue(queueName, mockMgr, scheduler, null); + schedulable = new FSLeafQueue(queueName, scheduler, null); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index e739a1452f4..a3aa3f6614d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -55,7 +55,6 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl; import org.apache.hadoop.yarn.api.records.NodeId; @@ -86,7 +85,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSc import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Clock; @@ -121,6 +119,7 @@ public class TestFairScheduler { private FairScheduler scheduler; private ResourceManager resourceManager; + private Configuration conf; private static RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); private int APP_ID = 1; // Incrementing counter for schedling apps @@ -130,7 +129,7 @@ public class TestFairScheduler { @Before public void setUp() throws IOException { scheduler = new FairScheduler(); - Configuration conf = createConfiguration(); + conf = createConfiguration(); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0); conf.setInt(FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1024); @@ -145,7 +144,6 @@ public class TestFairScheduler { ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start(); resourceManager.getRMContext().getStateStore().start(); - scheduler.reinitialize(conf, resourceManager.getRMContext()); // to initialize the master key resourceManager.getRMContainerTokenSecretManager().rollMasterKey(); } @@ -291,7 +289,6 @@ public class TestFairScheduler { @Test(timeout=2000) public void testLoadConfigurationOnInitialize() throws IOException { - Configuration conf = createConfiguration(); conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, true); conf.setInt(FairSchedulerConfiguration.MAX_ASSIGN, 3); conf.setBoolean(FairSchedulerConfiguration.SIZE_BASED_WEIGHT, true); @@ -362,6 +359,8 @@ public class TestFairScheduler { @Test public void testAggregateCapacityTracking() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + // Add a node RMNode node1 = MockNodes @@ -384,7 +383,9 @@ public class TestFairScheduler { } @Test - public void testSimpleFairShareCalculation() { + public void testSimpleFairShareCalculation() throws IOException { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + // Add one big node (only care about aggregate capacity) RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(10 * 1024), 1, @@ -409,7 +410,9 @@ public class TestFairScheduler { } @Test - public void testSimpleHierarchicalFairShareCalculation() { + public void testSimpleHierarchicalFairShareCalculation() throws IOException { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + // Add one big node (only care about aggregate capacity) int capacity = 10 * 24; RMNode node1 = @@ -440,7 +443,9 @@ public class TestFairScheduler { } @Test - public void testHierarchicalQueuesSimilarParents() { + public void testHierarchicalQueuesSimilarParents() throws IOException { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + QueueManager queueManager = scheduler.getQueueManager(); FSLeafQueue leafQueue = queueManager.getLeafQueue("parent.child", true); Assert.assertEquals(2, queueManager.getLeafQueues().size()); @@ -462,8 +467,9 @@ public class TestFairScheduler { } @Test - public void testSchedulerRootQueueMetrics() throws InterruptedException { - + public void testSchedulerRootQueueMetrics() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + // Add a node RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024)); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); @@ -500,7 +506,9 @@ public class TestFairScheduler { } @Test (timeout = 5000) - public void testSimpleContainerAllocation() { + public void testSimpleContainerAllocation() throws IOException { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + // Add a node RMNode node1 = MockNodes @@ -546,7 +554,9 @@ public class TestFairScheduler { } @Test (timeout = 5000) - public void testSimpleContainerReservation() throws InterruptedException { + public void testSimpleContainerReservation() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + // Add a node RMNode node1 = MockNodes @@ -598,7 +608,6 @@ public class TestFairScheduler { @Test public void testUserAsDefaultQueue() throws Exception { - Configuration conf = createConfiguration(); conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true"); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMContext rmContext = resourceManager.getRMContext(); @@ -617,14 +626,24 @@ public class TestFairScheduler { assertEquals(0, scheduler.getQueueManager().getLeafQueue("default", true) .getRunnableAppSchedulables().size()); assertEquals("root.user1", rmApp.getQueue()); - + } + + @Test + public void testNotUserAsDefaultQueue() throws Exception { conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "false"); scheduler.reinitialize(conf, resourceManager.getRMContext()); - scheduler.getQueueManager().initialize(); + RMContext rmContext = resourceManager.getRMContext(); + Map appsMap = rmContext.getRMApps(); + ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); + RMApp rmApp = new RMAppImpl(appAttemptId.getApplicationId(), rmContext, conf, + null, null, null, ApplicationSubmissionContext.newInstance(null, null, + null, null, null, false, false, 0, null, null), null, null, 0, null); + appsMap.put(appAttemptId.getApplicationId(), rmApp); + AppAddedSchedulerEvent appAddedEvent2 = new AppAddedSchedulerEvent( - createAppAttemptId(2, 1), "default", "user2"); + appAttemptId, "default", "user2"); scheduler.handle(appAddedEvent2); - assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true) + assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1", true) .getRunnableAppSchedulables().size()); assertEquals(1, scheduler.getQueueManager().getLeafQueue("default", true) .getRunnableAppSchedulables().size()); @@ -634,7 +653,7 @@ public class TestFairScheduler { @Test public void testEmptyQueueName() throws Exception { - Configuration conf = createConfiguration(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); // only default queue assertEquals(1, scheduler.getQueueManager().getLeafQueues().size()); @@ -653,7 +672,6 @@ public class TestFairScheduler { @Test public void testAssignToQueue() throws Exception { - Configuration conf = createConfiguration(); conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true"); scheduler.reinitialize(conf, resourceManager.getRMContext()); @@ -672,9 +690,10 @@ public class TestFairScheduler { @Test public void testQueuePlacementWithPolicy() throws Exception { - Configuration conf = createConfiguration(); conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, SimpleGroupsMapping.class, GroupMappingServiceProvider.class); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + ApplicationAttemptId appId; Map apps = scheduler.applications; @@ -684,10 +703,10 @@ public class TestFairScheduler { rules.add(new QueuePlacementRule.PrimaryGroup().initialize(false, null)); rules.add(new QueuePlacementRule.SecondaryGroupExistingQueue().initialize(false, null)); rules.add(new QueuePlacementRule.Default().initialize(true, null)); - Set queues = Sets.newHashSet("root.user1", "root.user3group", + Set queues = Sets.newHashSet("root.user1", "root.user3group", "root.user4subgroup1", "root.user4subgroup2" , "root.user5subgroup2"); - scheduler.getQueueManager().placementPolicy = new QueuePlacementPolicy( - rules, queues, conf); + scheduler.getAllocationConfiguration().placementPolicy = + new QueuePlacementPolicy(rules, queues, conf); appId = createSchedulingRequest(1024, "somequeue", "user1"); assertEquals("root.somequeue", apps.get(appId).getQueueName()); appId = createSchedulingRequest(1024, "default", "user1"); @@ -706,8 +725,8 @@ public class TestFairScheduler { rules.add(new QueuePlacementRule.User().initialize(false, null)); rules.add(new QueuePlacementRule.Specified().initialize(true, null)); rules.add(new QueuePlacementRule.Default().initialize(true, null)); - scheduler.getQueueManager().placementPolicy = new QueuePlacementPolicy( - rules, queues, conf); + scheduler.getAllocationConfiguration().placementPolicy = + new QueuePlacementPolicy(rules, queues, conf); appId = createSchedulingRequest(1024, "somequeue", "user1"); assertEquals("root.user1", apps.get(appId).getQueueName()); appId = createSchedulingRequest(1024, "somequeue", "otheruser"); @@ -718,9 +737,7 @@ public class TestFairScheduler { @Test public void testFairShareWithMinAlloc() throws Exception { - Configuration conf = createConfiguration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - scheduler.reinitialize(conf, resourceManager.getRMContext()); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println(""); @@ -733,9 +750,8 @@ public class TestFairScheduler { out.println(""); out.println(""); out.close(); - - QueueManager queueManager = scheduler.getQueueManager(); - queueManager.initialize(); + + scheduler.reinitialize(conf, resourceManager.getRMContext()); // Add one big node (only care about aggregate capacity) RMNode node1 = @@ -767,6 +783,8 @@ public class TestFairScheduler { */ @Test public void testQueueDemandCalculation() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + ApplicationAttemptId id11 = createAppAttemptId(1, 1); scheduler.addApplication(id11, "root.queue1", "user1"); ApplicationAttemptId id21 = createAppAttemptId(2, 1); @@ -812,6 +830,8 @@ public class TestFairScheduler { @Test public void testAppAdditionAndRemoval() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + AppAddedSchedulerEvent appAddedEvent1 = new AppAddedSchedulerEvent( createAppAttemptId(1, 1), "default", "user1"); scheduler.handle(appAddedEvent1); @@ -834,133 +854,10 @@ public class TestFairScheduler { .getRunnableAppSchedulables().size()); } - @Test - public void testAllocationFileParsing() throws Exception { - Configuration conf = createConfiguration(); - conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - scheduler.reinitialize(conf, resourceManager.getRMContext()); - - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - // Give queue A a minimum of 1024 M - out.println(""); - out.println("1024mb,0vcores"); - out.println(""); - // Give queue B a minimum of 2048 M - out.println(""); - out.println("2048mb,0vcores"); - out.println("alice,bob admins"); - out.println("fair"); - out.println(""); - // Give queue C no minimum - out.println(""); - out.println("alice,bob admins"); - out.println(""); - // Give queue D a limit of 3 running apps - out.println(""); - out.println("3"); - out.println(""); - // Give queue E a preemption timeout of one minute - out.println(""); - out.println("60"); - out.println(""); - // Set default limit of apps per queue to 15 - out.println("15"); - // Set default limit of apps per user to 5 - out.println("5"); - // Give user1 a limit of 10 jobs - out.println(""); - out.println("10"); - out.println(""); - // Set default min share preemption timeout to 2 minutes - out.println("120" - + ""); - // Set fair share preemption timeout to 5 minutes - out.println("300"); - // Set default scheduling policy to DRF - out.println("drf"); - out.println(""); - out.close(); - - QueueManager queueManager = scheduler.getQueueManager(); - queueManager.initialize(); - - assertEquals(6, queueManager.getLeafQueues().size()); // 5 in file + default queue - assertEquals(Resources.createResource(0), - queueManager.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); - assertEquals(Resources.createResource(0), - queueManager.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); - - assertEquals(Resources.createResource(1024, 0), - queueManager.getMinResources("root.queueA")); - assertEquals(Resources.createResource(2048, 0), - queueManager.getMinResources("root.queueB")); - assertEquals(Resources.createResource(0), - queueManager.getMinResources("root.queueC")); - assertEquals(Resources.createResource(0), - queueManager.getMinResources("root.queueD")); - assertEquals(Resources.createResource(0), - queueManager.getMinResources("root.queueE")); - - assertEquals(15, queueManager.getQueueMaxApps("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); - assertEquals(15, queueManager.getQueueMaxApps("root.queueA")); - assertEquals(15, queueManager.getQueueMaxApps("root.queueB")); - assertEquals(15, queueManager.getQueueMaxApps("root.queueC")); - assertEquals(3, queueManager.getQueueMaxApps("root.queueD")); - assertEquals(15, queueManager.getQueueMaxApps("root.queueE")); - assertEquals(10, queueManager.getUserMaxApps("user1")); - assertEquals(5, queueManager.getUserMaxApps("user2")); - - // Root should get * ACL - assertEquals("*",queueManager.getQueueAcl("root", - QueueACL.ADMINISTER_QUEUE).getAclString()); - assertEquals("*", queueManager.getQueueAcl("root", - QueueACL.SUBMIT_APPLICATIONS).getAclString()); - - // Unspecified queues should get default ACL - assertEquals(" ",queueManager.getQueueAcl("root.queueA", - QueueACL.ADMINISTER_QUEUE).getAclString()); - assertEquals(" ", queueManager.getQueueAcl("root.queueA", - QueueACL.SUBMIT_APPLICATIONS).getAclString()); - - // Queue B ACL - assertEquals("alice,bob admins",queueManager.getQueueAcl("root.queueB", - QueueACL.ADMINISTER_QUEUE).getAclString()); - - // Queue C ACL - assertEquals("alice,bob admins",queueManager.getQueueAcl("root.queueC", - QueueACL.SUBMIT_APPLICATIONS).getAclString()); - - assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root." + - YarnConfiguration.DEFAULT_QUEUE_NAME)); - assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueA")); - assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueB")); - assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueC")); - assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueD")); - assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueA")); - assertEquals(60000, queueManager.getMinSharePreemptionTimeout("root.queueE")); - assertEquals(300000, queueManager.getFairSharePreemptionTimeout()); - - // Verify existing queues have default scheduling policy - assertEquals(DominantResourceFairnessPolicy.NAME, - queueManager.getQueue("root").getPolicy().getName()); - assertEquals(DominantResourceFairnessPolicy.NAME, - queueManager.getQueue("root.queueA").getPolicy().getName()); - // Verify default is overriden if specified explicitly - assertEquals(FairSharePolicy.NAME, - queueManager.getQueue("root.queueB").getPolicy().getName()); - // Verify new queue gets default scheduling policy - assertEquals(DominantResourceFairnessPolicy.NAME, - queueManager.getLeafQueue("root.newqueue", true).getPolicy().getName()); - } - @Test public void testHierarchicalQueueAllocationFileParsing() throws IOException, SAXException, AllocationConfigurationException, ParserConfigurationException { - Configuration conf = createConfiguration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - scheduler.reinitialize(conf, resourceManager.getRMContext()); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println(""); @@ -980,9 +877,9 @@ public class TestFairScheduler { out.println(""); out.close(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + QueueManager queueManager = scheduler.getQueueManager(); - queueManager.initialize(); - Collection leafQueues = queueManager.getLeafQueues(); Assert.assertEquals(4, leafQueues.size()); Assert.assertNotNull(queueManager.getLeafQueue("queueA", false)); @@ -995,9 +892,7 @@ public class TestFairScheduler { @Test public void testConfigureRootQueue() throws Exception { - Configuration conf = createConfiguration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - scheduler.reinitialize(conf, resourceManager.getRMContext()); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println(""); @@ -1014,9 +909,9 @@ public class TestFairScheduler { out.println(""); out.println(""); out.close(); - + + scheduler.reinitialize(conf, resourceManager.getRMContext()); QueueManager queueManager = scheduler.getQueueManager(); - queueManager.initialize(); FSQueue root = queueManager.getRootQueue(); assertTrue(root.getPolicy() instanceof DominantResourceFairnessPolicy); @@ -1025,136 +920,9 @@ public class TestFairScheduler { assertNotNull(queueManager.getLeafQueue("child2", false)); } - /** - * Verify that you can't place queues at the same level as the root queue in - * the allocations file. - */ - @Test (expected = AllocationConfigurationException.class) - public void testQueueAlongsideRoot() throws Exception { - Configuration conf = createConfiguration(); - conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - scheduler.reinitialize(conf, resourceManager.getRMContext()); - - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.close(); - - QueueManager queueManager = scheduler.getQueueManager(); - queueManager.initialize(); - } - - @Test - public void testBackwardsCompatibleAllocationFileParsing() throws Exception { - Configuration conf = createConfiguration(); - conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - scheduler.reinitialize(conf, resourceManager.getRMContext()); - - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - // Give queue A a minimum of 1024 M - out.println(""); - out.println("1024mb,0vcores"); - out.println(""); - // Give queue B a minimum of 2048 M - out.println(""); - out.println("2048mb,0vcores"); - out.println("alice,bob admins"); - out.println(""); - // Give queue C no minimum - out.println(""); - out.println("alice,bob admins"); - out.println(""); - // Give queue D a limit of 3 running apps - out.println(""); - out.println("3"); - out.println(""); - // Give queue E a preemption timeout of one minute - out.println(""); - out.println("60"); - out.println(""); - // Set default limit of apps per queue to 15 - out.println("15"); - // Set default limit of apps per user to 5 - out.println("5"); - // Give user1 a limit of 10 jobs - out.println(""); - out.println("10"); - out.println(""); - // Set default min share preemption timeout to 2 minutes - out.println("120" - + ""); - // Set fair share preemption timeout to 5 minutes - out.println("300"); - out.println(""); - out.close(); - - QueueManager queueManager = scheduler.getQueueManager(); - queueManager.initialize(); - - assertEquals(6, queueManager.getLeafQueues().size()); // 5 in file + default queue - assertEquals(Resources.createResource(0), - queueManager.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); - assertEquals(Resources.createResource(0), - queueManager.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); - - assertEquals(Resources.createResource(1024, 0), - queueManager.getMinResources("root.queueA")); - assertEquals(Resources.createResource(2048, 0), - queueManager.getMinResources("root.queueB")); - assertEquals(Resources.createResource(0), - queueManager.getMinResources("root.queueC")); - assertEquals(Resources.createResource(0), - queueManager.getMinResources("root.queueD")); - assertEquals(Resources.createResource(0), - queueManager.getMinResources("root.queueE")); - - assertEquals(15, queueManager.getQueueMaxApps("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); - assertEquals(15, queueManager.getQueueMaxApps("root.queueA")); - assertEquals(15, queueManager.getQueueMaxApps("root.queueB")); - assertEquals(15, queueManager.getQueueMaxApps("root.queueC")); - assertEquals(3, queueManager.getQueueMaxApps("root.queueD")); - assertEquals(15, queueManager.getQueueMaxApps("root.queueE")); - assertEquals(10, queueManager.getUserMaxApps("user1")); - assertEquals(5, queueManager.getUserMaxApps("user2")); - - // Unspecified queues should get default ACL - assertEquals(" ", queueManager.getQueueAcl("root.queueA", - QueueACL.ADMINISTER_QUEUE).getAclString()); - assertEquals(" ", queueManager.getQueueAcl("root.queueA", - QueueACL.SUBMIT_APPLICATIONS).getAclString()); - - // Queue B ACL - assertEquals("alice,bob admins", queueManager.getQueueAcl("root.queueB", - QueueACL.ADMINISTER_QUEUE).getAclString()); - - // Queue C ACL - assertEquals("alice,bob admins", queueManager.getQueueAcl("root.queueC", - QueueACL.SUBMIT_APPLICATIONS).getAclString()); - - - assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root." + - YarnConfiguration.DEFAULT_QUEUE_NAME)); - assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueA")); - assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueB")); - assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueC")); - assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueD")); - assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueA")); - assertEquals(60000, queueManager.getMinSharePreemptionTimeout("root.queueE")); - assertEquals(300000, queueManager.getFairSharePreemptionTimeout()); - } - @Test (timeout = 5000) public void testIsStarvedForMinShare() throws Exception { - Configuration conf = createConfiguration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - scheduler.reinitialize(conf, resourceManager.getRMContext()); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println(""); @@ -1168,8 +936,7 @@ public class TestFairScheduler { out.println(""); out.close(); - QueueManager queueManager = scheduler.getQueueManager(); - queueManager.initialize(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); // Add one big node (only care about aggregate capacity) RMNode node1 = @@ -1212,9 +979,7 @@ public class TestFairScheduler { @Test (timeout = 5000) public void testIsStarvedForFairShare() throws Exception { - Configuration conf = createConfiguration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - scheduler.reinitialize(conf, resourceManager.getRMContext()); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println(""); @@ -1228,9 +993,8 @@ public class TestFairScheduler { out.println(""); out.close(); - QueueManager queueManager = scheduler.getQueueManager(); - queueManager.initialize(); - + scheduler.reinitialize(conf, resourceManager.getRMContext()); + // Add one big node (only care about aggregate capacity) RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(4 * 1024, 4), 1, @@ -1277,13 +1041,9 @@ public class TestFairScheduler { * now this means decreasing order of priority. */ public void testChoiceOfPreemptedContainers() throws Exception { - Configuration conf = createConfiguration(); - conf.setLong(FairSchedulerConfiguration.PREEMPTION_INTERVAL, 5000); conf.setLong(FairSchedulerConfiguration.WAIT_TIME_BEFORE_KILL, 10000); - conf.set(FairSchedulerConfiguration.ALLOCATION_FILE + ".allocation.file", ALLOC_FILE); - scheduler.reinitialize(conf, resourceManager.getRMContext()); MockClock clock = new MockClock(); scheduler.setClock(clock); @@ -1305,9 +1065,8 @@ public class TestFairScheduler { out.println(""); out.println(""); out.close(); - - QueueManager queueManager = scheduler.getQueueManager(); - queueManager.initialize(); + + scheduler.reinitialize(conf, resourceManager.getRMContext()); // Create four nodes RMNode node1 = @@ -1443,15 +1202,16 @@ public class TestFairScheduler { * Tests the timing of decision to preempt tasks. */ public void testPreemptionDecision() throws Exception { - Configuration conf = createConfiguration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); MockClock clock = new MockClock(); scheduler.setClock(clock); - scheduler.reinitialize(conf, resourceManager.getRMContext()); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println(""); out.println(""); + out.println(""); + out.println("0mb,0vcores"); + out.println(""); out.println(""); out.println(".25"); out.println("1024mb,0vcores"); @@ -1473,8 +1233,7 @@ public class TestFairScheduler { out.println(""); out.close(); - QueueManager queueManager = scheduler.getQueueManager(); - queueManager.initialize(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); // Create four nodes RMNode node1 = @@ -1570,7 +1329,9 @@ public class TestFairScheduler { } @Test (timeout = 5000) - public void testMultipleContainersWaitingForReservation() { + public void testMultipleContainersWaitingForReservation() throws IOException { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + // Add a node RMNode node1 = MockNodes @@ -1600,9 +1361,7 @@ public class TestFairScheduler { @Test (timeout = 5000) public void testUserMaxRunningApps() throws Exception { // Set max running apps - Configuration conf = createConfiguration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - scheduler.reinitialize(conf, resourceManager.getRMContext()); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println(""); @@ -1613,8 +1372,7 @@ public class TestFairScheduler { out.println(""); out.close(); - QueueManager queueManager = scheduler.getQueueManager(); - queueManager.initialize(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); // Add a node RMNode node1 = @@ -1654,7 +1412,9 @@ public class TestFairScheduler { } @Test (timeout = 5000) - public void testReservationWhileMultiplePriorities() { + public void testReservationWhileMultiplePriorities() throws IOException { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + // Add a node RMNode node1 = MockNodes @@ -1717,9 +1477,7 @@ public class TestFairScheduler { @Test public void testAclSubmitApplication() throws Exception { // Set acl's - Configuration conf = createConfiguration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - scheduler.reinitialize(conf, resourceManager.getRMContext()); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println(""); @@ -1735,8 +1493,7 @@ public class TestFairScheduler { out.println(""); out.close(); - QueueManager queueManager = scheduler.getQueueManager(); - queueManager.initialize(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1", "norealuserhasthisname", 1); @@ -1751,6 +1508,8 @@ public class TestFairScheduler { @Test (timeout = 5000) public void testMultipleNodesSingleRackRequest() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + RMNode node1 = MockNodes .newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1"); @@ -1797,6 +1556,8 @@ public class TestFairScheduler { @Test (timeout = 5000) public void testFifoWithinQueue() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + RMNode node1 = MockNodes .newNodeInfo(1, Resources.createResource(3072, 3), 1, "127.0.0.1"); @@ -1837,11 +1598,9 @@ public class TestFairScheduler { } @Test(timeout = 3000) - public void testMaxAssign() throws AllocationConfigurationException { - // set required scheduler configs - scheduler.assignMultiple = true; - scheduler.getQueueManager().getLeafQueue("root.default", true) - .setPolicy(SchedulingPolicy.getDefault()); + public void testMaxAssign() throws Exception { + conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, true); + scheduler.reinitialize(conf, resourceManager.getRMContext()); RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(16384, 16), 0, @@ -1884,6 +1643,8 @@ public class TestFairScheduler { */ @Test(timeout = 5000) public void testAssignContainer() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + final String user = "user1"; final String fifoQueue = "fifo"; final String fairParent = "fairParent"; @@ -1951,9 +1712,7 @@ public class TestFairScheduler { @Test public void testNotAllowSubmitApplication() throws Exception { // Set acl's - Configuration conf = createConfiguration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - scheduler.reinitialize(conf, resourceManager.getRMContext()); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println(""); out.println(""); @@ -1967,8 +1726,8 @@ public class TestFairScheduler { out.println(""); out.println(""); out.close(); - QueueManager queueManager = scheduler.getQueueManager(); - queueManager.initialize(); + + scheduler.reinitialize(conf, resourceManager.getRMContext()); int appId = this.APP_ID++; String user = "usernotallow"; @@ -2017,7 +1776,9 @@ public class TestFairScheduler { } @Test - public void testReservationThatDoesntFit() { + public void testReservationThatDoesntFit() throws IOException { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + RMNode node1 = MockNodes .newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1"); @@ -2043,7 +1804,9 @@ public class TestFairScheduler { } @Test - public void testRemoveNodeUpdatesRootQueueMetrics() { + public void testRemoveNodeUpdatesRootQueueMetrics() throws IOException { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB()); assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); @@ -2069,7 +1832,9 @@ public class TestFairScheduler { } @Test - public void testStrictLocality() { + public void testStrictLocality() throws IOException { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); @@ -2107,7 +1872,9 @@ public class TestFairScheduler { } @Test - public void testCancelStrictLocality() { + public void testCancelStrictLocality() throws IOException { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); @@ -2155,7 +1922,9 @@ public class TestFairScheduler { * a reservation on another. */ @Test - public void testReservationsStrictLocality() { + public void testReservationsStrictLocality() throws IOException { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1"); RMNode node2 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 2, "127.0.0.2"); NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node1); @@ -2193,7 +1962,9 @@ public class TestFairScheduler { } @Test - public void testNoMoreCpuOnNode() { + public void testNoMoreCpuOnNode() throws IOException { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 1), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); @@ -2213,6 +1984,8 @@ public class TestFairScheduler { @Test public void testBasicDRFAssignment() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + RMNode node = MockNodes.newNodeInfo(1, BuilderUtils.newResource(8192, 5)); NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node); scheduler.handle(nodeEvent); @@ -2251,6 +2024,8 @@ public class TestFairScheduler { */ @Test public void testBasicDRFWithQueues() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + RMNode node = MockNodes.newNodeInfo(1, BuilderUtils.newResource(8192, 7), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node); @@ -2285,6 +2060,8 @@ public class TestFairScheduler { @Test public void testDRFHierarchicalQueues() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + RMNode node = MockNodes.newNodeInfo(1, BuilderUtils.newResource(12288, 12), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node); @@ -2349,9 +2126,9 @@ public class TestFairScheduler { @Test(timeout = 30000) public void testHostPortNodeName() throws Exception { - scheduler.getConf().setBoolean(YarnConfiguration + conf.setBoolean(YarnConfiguration .RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, true); - scheduler.reinitialize(scheduler.getConf(), + scheduler.reinitialize(conf, resourceManager.getRMContext()); RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1", 1); @@ -2426,9 +2203,7 @@ public class TestFairScheduler { @Test public void testUserAndQueueMaxRunningApps() throws Exception { - Configuration conf = createConfiguration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - scheduler.reinitialize(conf, resourceManager.getRMContext()); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println(""); @@ -2442,8 +2217,7 @@ public class TestFairScheduler { out.println(""); out.close(); - QueueManager queueManager = scheduler.getQueueManager(); - queueManager.initialize(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); // exceeds no limits ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1", "user1"); @@ -2479,9 +2253,7 @@ public class TestFairScheduler { @Test public void testMaxRunningAppsHierarchicalQueues() throws Exception { - Configuration conf = createConfiguration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - scheduler.reinitialize(conf, resourceManager.getRMContext()); MockClock clock = new MockClock(); scheduler.setClock(clock); @@ -2499,8 +2271,7 @@ public class TestFairScheduler { out.println(""); out.close(); - QueueManager queueManager = scheduler.getQueueManager(); - queueManager.initialize(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); // exceeds no limits ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1.sub1", "user1"); @@ -2629,10 +2400,8 @@ public class TestFairScheduler { @Test public void testDontAllowUndeclaredPools() throws Exception{ - Configuration conf = createConfiguration(); conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - scheduler.reinitialize(conf, resourceManager.getRMContext()); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println(""); @@ -2642,8 +2411,8 @@ public class TestFairScheduler { out.println(""); out.close(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); QueueManager queueManager = scheduler.getQueueManager(); - queueManager.initialize(); FSLeafQueue jerryQueue = queueManager.getLeafQueue("jerry", false); FSLeafQueue defaultQueue = queueManager.getLeafQueue("default", false); @@ -2672,6 +2441,8 @@ public class TestFairScheduler { @SuppressWarnings("resource") @Test public void testBlacklistNodes() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + final int GB = 1024; String host = "127.0.0.1"; RMNode node = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java index da8a183c10a..85e1cb97144 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java @@ -61,14 +61,4 @@ public class TestFairSchedulerConfiguration { parseResourceConfigValue("1o24vc0res"); } - @Test - public void testGetAllocationFileFromClasspath() { - FairSchedulerConfiguration conf = new FairSchedulerConfiguration( - new Configuration()); - conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, - "test-fair-scheduler.xml"); - File allocationFile = conf.getAllocationFile(); - Assert.assertEquals("test-fair-scheduler.xml", allocationFile.getName()); - Assert.assertTrue(allocationFile.exists()); - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java index 20f6e3d7757..e70c039dcaa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java @@ -39,18 +39,21 @@ public class TestMaxRunningAppsEnforcer { @Before public void setup() throws Exception { + Configuration conf = new Configuration(); clock = new TestFairScheduler.MockClock(); FairScheduler scheduler = mock(FairScheduler.class); when(scheduler.getConf()).thenReturn( - new FairSchedulerConfiguration(new Configuration())); + new FairSchedulerConfiguration(conf)); when(scheduler.getClock()).thenReturn(clock); + AllocationConfiguration allocConf = new AllocationConfiguration( + conf); + when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); queueManager = new QueueManager(scheduler); - queueManager.initialize(); - - queueMaxApps = queueManager.info.queueMaxApps; - userMaxApps = queueManager.info.userMaxApps; - maxAppsEnforcer = new MaxRunningAppsEnforcer(queueManager); + queueManager.initialize(conf); + queueMaxApps = allocConf.queueMaxApps; + userMaxApps = allocConf.userMaxApps; + maxAppsEnforcer = new MaxRunningAppsEnforcer(scheduler); appNum = 0; } From aa4fba6d92faf5e51bb330c6d18825fbed63b553 Mon Sep 17 00:00:00 2001 From: Brandon Li Date: Thu, 5 Dec 2013 07:13:21 +0000 Subject: [PATCH 17/32] HDFS-5587. add debug information when NFS fails to start with duplicate user or group names. Contributed by Brandon Li git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1548028 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/hadoop/nfs/nfs3/IdUserGroup.java | 87 ++++++++++++++----- .../hadoop/nfs/nfs3/TestIdUserGroup.java | 56 ++++++++++++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + 3 files changed, 124 insertions(+), 22 deletions(-) create mode 100644 hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java index e034c664059..a1d48aadc85 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java @@ -24,6 +24,7 @@ import java.io.InputStreamReader; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; @@ -44,13 +45,21 @@ public class IdUserGroup { // Do update every 15 minutes final static long TIMEOUT = 15 * 60 * 1000; // ms - // Maps for id to name map. Guarded by this object monitor lock */ + // Maps for id to name map. Guarded by this object monitor lock private BiMap uidNameMap = HashBiMap.create(); private BiMap gidNameMap = HashBiMap.create(); private long lastUpdateTime = 0; // Last time maps were updated - public IdUserGroup() { + static public class DuplicateNameOrIdException extends IOException { + private static final long serialVersionUID = 1L; + + public DuplicateNameOrIdException(String msg) { + super(msg); + } + } + + public IdUserGroup() throws IOException { updateMaps(); } @@ -58,18 +67,34 @@ public class IdUserGroup { return lastUpdateTime - System.currentTimeMillis() > TIMEOUT; } + // If can't update the maps, will keep using the old ones private void checkAndUpdateMaps() { if (isExpired()) { LOG.info("Update cache now"); - updateMaps(); + try { + updateMaps(); + } catch (IOException e) { + LOG.error("Can't update the maps. Will use the old ones," + + " which can potentially cause problem.", e); + } } } + private static final String DUPLICATE_NAME_ID_DEBUG_INFO = "NFS gateway can't start with duplicate name or id on the host system.\n" + + "This is because HDFS (non-kerberos cluster) uses name as the only way to identify a user or group.\n" + + "The host system with duplicated user/group name or id might work fine most of the time by itself.\n" + + "However when NFS gateway talks to HDFS, HDFS accepts only user and group name.\n" + + "Therefore, same name means the same user or same group. To find the duplicated names/ids, one can do:\n" + + " and on Linux systms,\n" + + " and on MacOS."; + /** * Get the whole list of users and groups and save them in the maps. + * @throws IOException */ - private void updateMapInternal(BiMap map, String name, - String command, String regex) throws IOException { + @VisibleForTesting + public static void updateMapInternal(BiMap map, String mapName, + String command, String regex) throws IOException { BufferedReader br = null; try { Process process = Runtime.getRuntime().exec( @@ -79,15 +104,31 @@ public class IdUserGroup { while ((line = br.readLine()) != null) { String[] nameId = line.split(regex); if ((nameId == null) || (nameId.length != 2)) { - throw new IOException("Can't parse " + name + " list entry:" + line); + throw new IOException("Can't parse " + mapName + " list entry:" + line); + } + LOG.debug("add to " + mapName + "map:" + nameId[0] + " id:" + nameId[1]); + // HDFS can't differentiate duplicate names with simple authentication + Integer key = Integer.valueOf(nameId[1]); + String value = nameId[0]; + if (map.containsKey(key)) { + LOG.error(String.format( + "Got duplicate id:(%d, %s), existing entry: (%d, %s).\n%s", key, + value, key, map.get(key), DUPLICATE_NAME_ID_DEBUG_INFO)); + throw new DuplicateNameOrIdException("Got duplicate id."); + } + if (map.containsValue(nameId[0])) { + LOG.error(String.format( + "Got duplicate name:(%d, %s), existing entry: (%d, %s) \n%s", + key, value, map.inverse().get(value), value, + DUPLICATE_NAME_ID_DEBUG_INFO)); + throw new DuplicateNameOrIdException("Got duplicate name"); } - LOG.debug("add " + name + ":" + nameId[0] + " id:" + nameId[1]); map.put(Integer.valueOf(nameId[1]), nameId[0]); } - LOG.info("Updated " + name + " map size:" + map.size()); + LOG.info("Updated " + mapName + " map size:" + map.size()); } catch (IOException e) { - LOG.error("Can't update map " + name); + LOG.error("Can't update " + mapName + " map"); throw e; } finally { if (br != null) { @@ -101,24 +142,26 @@ public class IdUserGroup { } } - synchronized public void updateMaps() { + synchronized public void updateMaps() throws IOException { BiMap uMap = HashBiMap.create(); BiMap gMap = HashBiMap.create(); - try { - if (OS.startsWith("Linux")) { - updateMapInternal(uMap, "user", LINUX_GET_ALL_USERS_CMD, ":"); - updateMapInternal(gMap, "group", LINUX_GET_ALL_GROUPS_CMD, ":"); - } else if (OS.startsWith("Mac")) { - updateMapInternal(uMap, "user", MAC_GET_ALL_USERS_CMD, "\\s+"); - updateMapInternal(gMap, "group", MAC_GET_ALL_GROUPS_CMD, "\\s+"); - } else { - throw new IOException("Platform is not supported:" + OS); - } - } catch (IOException e) { - LOG.error("Can't update maps:" + e); + if (!OS.startsWith("Linux") && !OS.startsWith("Mac")) { + LOG.error("Platform is not supported:" + OS + + ". Can't update user map and group map and" + + " 'nobody' will be used for any user and group."); return; } + + if (OS.startsWith("Linux")) { + updateMapInternal(uMap, "user", LINUX_GET_ALL_USERS_CMD, ":"); + updateMapInternal(gMap, "group", LINUX_GET_ALL_GROUPS_CMD, ":"); + } else { + // Mac + updateMapInternal(uMap, "user", MAC_GET_ALL_USERS_CMD, "\\s+"); + updateMapInternal(gMap, "group", MAC_GET_ALL_GROUPS_CMD, "\\s+"); + } + uidNameMap = uMap; gidNameMap = gMap; lastUpdateTime = System.currentTimeMillis(); diff --git a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java new file mode 100644 index 00000000000..db2b27016e4 --- /dev/null +++ b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.nfs.nfs3; + +import static org.junit.Assert.fail; + +import java.io.IOException; + +import org.apache.hadoop.nfs.nfs3.IdUserGroup.DuplicateNameOrIdException; +import org.junit.Test; + +import com.google.common.collect.BiMap; +import com.google.common.collect.HashBiMap; + +public class TestIdUserGroup { + + @Test + public void testDuplicates() throws IOException { + String GET_ALL_USERS_CMD = "echo \"root:x:0:0:root:/root:/bin/bash\n" + + "hdfs:x:11501:10787:Grid Distributed File System:/home/hdfs:/bin/bash\n" + + "hdfs:x:11502:10788:Grid Distributed File System:/home/hdfs:/bin/bash\"" + + " | cut -d: -f1,3"; + String GET_ALL_GROUPS_CMD = "echo \"hdfs:*:11501:hrt_hdfs\n" + + "mapred:x:497\n" + "mapred2:x:497\"" + " | cut -d: -f1,3"; + // Maps for id to name map + BiMap uMap = HashBiMap.create(); + BiMap gMap = HashBiMap.create(); + + try { + IdUserGroup.updateMapInternal(uMap, "user", GET_ALL_USERS_CMD, ":"); + fail("didn't detect the duplicate name"); + } catch (DuplicateNameOrIdException e) { + } + + try { + IdUserGroup.updateMapInternal(gMap, "group", GET_ALL_GROUPS_CMD, ":"); + fail("didn't detect the duplicate id"); + } catch (DuplicateNameOrIdException e) { + } + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index bc98abd88f3..d7ff8fb7a72 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -777,6 +777,9 @@ Release 2.3.0 - UNRELEASED HDFS-4997. libhdfs doesn't return correct error codes in most cases (cmccabe) + HDFS-5587. add debug information when NFS fails to start with duplicate user + or group names (brandonli) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES From 6828337d855abe8564ece2266ace7cfc51ee16ec Mon Sep 17 00:00:00 2001 From: Daryn Sharp Date: Thu, 5 Dec 2013 15:28:12 +0000 Subject: [PATCH 18/32] HDFS-5514. FSNamesystem's fsLock should allow custom implementation (daryn) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1548161 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../hdfs/server/namenode/FSNamesystem.java | 8 ++--- .../server/namenode/TestFSNamesystem.java | 36 +++++++++++++++++++ 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index d7ff8fb7a72..379b01af436 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -457,6 +457,8 @@ Release 2.4.0 - UNRELEASED HDFS-5444. Choose default web UI based on browser capabilities. (Haohui Mai via jing9) + HDFS-5514. FSNamesystem's fsLock should allow custom implementation (daryn) + IMPROVEMENTS HDFS-5267. Remove volatile from LightWeightHashSet. (Junping Du via llu) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 0cae09863b8..6a74b49fa53 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -467,7 +467,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, private final long accessTimePrecision; /** Lock to protect FSNamesystem. */ - private ReentrantReadWriteLock fsLock; + private FSNamesystemLock fsLock; /** * Used when this NN is in standby state to read from the shared edit log. @@ -650,7 +650,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, throws IOException { boolean fair = conf.getBoolean("dfs.namenode.fslock.fair", true); LOG.info("fsLock is fair:" + fair); - fsLock = new ReentrantReadWriteLock(fair); + fsLock = new FSNamesystemLock(fair); try { resourceRecheckInterval = conf.getLong( DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY, @@ -6771,12 +6771,12 @@ public class FSNamesystem implements Namesystem, FSClusterStats, @VisibleForTesting void setFsLockForTests(ReentrantReadWriteLock lock) { - this.fsLock = lock; + this.fsLock.coarseLock = lock; } @VisibleForTesting ReentrantReadWriteLock getFsLockForTests() { - return fsLock; + return fsLock.coarseLock; } @VisibleForTesting diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java index a2bd172d5e1..3af20a7303f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java @@ -158,4 +158,40 @@ public class TestFSNamesystem { fsNamesystem = new FSNamesystem(conf, fsImage); assertFalse(fsNamesystem.getFsLockForTests().isFair()); } + + @Test + public void testFSNamesystemLockCompatibility() { + FSNamesystemLock rwLock = new FSNamesystemLock(true); + + assertEquals(0, rwLock.getReadHoldCount()); + rwLock.readLock().lock(); + assertEquals(1, rwLock.getReadHoldCount()); + + rwLock.readLock().lock(); + assertEquals(2, rwLock.getReadHoldCount()); + + rwLock.readLock().unlock(); + assertEquals(1, rwLock.getReadHoldCount()); + + rwLock.readLock().unlock(); + assertEquals(0, rwLock.getReadHoldCount()); + + assertFalse(rwLock.isWriteLockedByCurrentThread()); + assertEquals(0, rwLock.getWriteHoldCount()); + rwLock.writeLock().lock(); + assertTrue(rwLock.isWriteLockedByCurrentThread()); + assertEquals(1, rwLock.getWriteHoldCount()); + + rwLock.writeLock().lock(); + assertTrue(rwLock.isWriteLockedByCurrentThread()); + assertEquals(2, rwLock.getWriteHoldCount()); + + rwLock.writeLock().unlock(); + assertTrue(rwLock.isWriteLockedByCurrentThread()); + assertEquals(1, rwLock.getWriteHoldCount()); + + rwLock.writeLock().unlock(); + assertFalse(rwLock.isWriteLockedByCurrentThread()); + assertEquals(0, rwLock.getWriteHoldCount()); + } } From 99aed805f8095972d8033ce90e24a9c7cef1eec5 Mon Sep 17 00:00:00 2001 From: Daryn Sharp Date: Thu, 5 Dec 2013 15:31:35 +0000 Subject: [PATCH 19/32] Neglected to add new file in HDFS-5514 (daryn) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1548167 13f79535-47bb-0310-9956-ffa450edef68 --- .../server/namenode/FSNamesystemLock.java | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java new file mode 100644 index 00000000000..a2a1648e458 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java @@ -0,0 +1,61 @@ + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.server.namenode; + +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import com.google.common.annotations.VisibleForTesting; + +/** + * Mimics a ReentrantReadWriteLock so more sophisticated locking capabilities + * are possible. + */ +class FSNamesystemLock implements ReadWriteLock { + @VisibleForTesting + protected ReentrantReadWriteLock coarseLock; + + FSNamesystemLock(boolean fair) { + this.coarseLock = new ReentrantReadWriteLock(fair); + } + + @Override + public Lock readLock() { + return coarseLock.readLock(); + } + + @Override + public Lock writeLock() { + return coarseLock.writeLock(); + } + + public int getReadHoldCount() { + return coarseLock.getReadHoldCount(); + } + + public int getWriteHoldCount() { + return coarseLock.getWriteHoldCount(); + } + + public boolean isWriteLockedByCurrentThread() { + return coarseLock.isWriteLockedByCurrentThread(); + } +} \ No newline at end of file From 9ea61e44153b938309841b1499488360e9abd176 Mon Sep 17 00:00:00 2001 From: Daryn Sharp Date: Thu, 5 Dec 2013 15:47:55 +0000 Subject: [PATCH 20/32] HADOOP-10129. Distcp may succeed when it fails (daryn) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1548175 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 4 ++ hadoop-tools/hadoop-distcp/pom.xml | 5 ++ .../hadoop/tools/SimpleCopyListing.java | 17 ++++-- .../mapred/RetriableFileCopyCommand.java | 7 ++- .../apache/hadoop/tools/TestCopyListing.java | 28 +++++++++ .../mapred/TestRetriableFileCopyCommand.java | 59 +++++++++++++++++++ 6 files changed, 113 insertions(+), 7 deletions(-) create mode 100644 hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestRetriableFileCopyCommand.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index a73b182635c..b8576d7e759 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -460,6 +460,8 @@ Release 2.4.0 - UNRELEASED HADOOP-10135 writes to swift fs over partition size leave temp files and empty output file (David Dobbins via stevel) + HADOOP-10129. Distcp may succeed when it fails (daryn) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES @@ -2355,6 +2357,8 @@ Release 0.23.10 - UNRELEASED BUG FIXES + HADOOP-10129. Distcp may succeed when it fails (daryn) + Release 0.23.9 - 2013-07-08 INCOMPATIBLE CHANGES diff --git a/hadoop-tools/hadoop-distcp/pom.xml b/hadoop-tools/hadoop-distcp/pom.xml index 1768a265d5f..9284592ca66 100644 --- a/hadoop-tools/hadoop-distcp/pom.xml +++ b/hadoop-tools/hadoop-distcp/pom.xml @@ -95,6 +95,11 @@ test test-jar + + org.mockito + mockito-all + test + diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java index 08552fbd603..c494995fb40 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java @@ -32,6 +32,8 @@ import org.apache.hadoop.tools.util.DistCpUtils; import org.apache.hadoop.mapreduce.security.TokenCache; import org.apache.hadoop.security.Credentials; +import com.google.common.annotations.VisibleForTesting; + import java.io.*; import java.util.Stack; @@ -107,12 +109,13 @@ public class SimpleCopyListing extends CopyListing { /** {@inheritDoc} */ @Override public void doBuildListing(Path pathToListingFile, DistCpOptions options) throws IOException { - - SequenceFile.Writer fileListWriter = null; - + doBuildListing(getWriter(pathToListingFile), options); + } + + @VisibleForTesting + public void doBuildListing(SequenceFile.Writer fileListWriter, + DistCpOptions options) throws IOException { try { - fileListWriter = getWriter(pathToListingFile); - for (Path path: options.getSourcePaths()) { FileSystem sourceFS = path.getFileSystem(getConf()); path = makeQualified(path); @@ -143,8 +146,10 @@ public class SimpleCopyListing extends CopyListing { localFile, options); } } + fileListWriter.close(); + fileListWriter = null; } finally { - IOUtils.closeStream(fileListWriter); + IOUtils.cleanup(LOG, fileListWriter); } } diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java index 87fb2d4511e..580229cf8e8 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java @@ -30,6 +30,8 @@ import org.apache.hadoop.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import com.google.common.annotations.VisibleForTesting; + import java.io.*; import java.util.EnumSet; @@ -176,7 +178,8 @@ public class RetriableFileCopyCommand extends RetriableCommand { return new Path(root, ".distcp.tmp." + context.getTaskAttemptID().toString()); } - private long copyBytes(FileStatus sourceFileStatus, OutputStream outStream, + @VisibleForTesting + long copyBytes(FileStatus sourceFileStatus, OutputStream outStream, int bufferSize, Mapper.Context context) throws IOException { Path source = sourceFileStatus.getPath(); @@ -193,6 +196,8 @@ public class RetriableFileCopyCommand extends RetriableCommand { updateContextStatus(totalBytesRead, context, sourceFileStatus); bytesRead = inStream.read(buf); } + outStream.close(); + outStream = null; } finally { IOUtils.cleanup(LOG, outStream, inStream); } diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java index fb327c32f03..11cf7821e39 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java @@ -18,6 +18,8 @@ package org.apache.hadoop.tools; +import static org.mockito.Mockito.*; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; @@ -36,6 +38,7 @@ import org.junit.Assert; import org.junit.BeforeClass; import org.junit.AfterClass; +import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.util.List; @@ -282,4 +285,29 @@ public class TestCopyListing extends SimpleCopyListing { IOUtils.closeStream(reader); } } + + @Test + public void testFailOnCloseError() throws IOException { + File inFile = File.createTempFile("TestCopyListingIn", null); + inFile.deleteOnExit(); + File outFile = File.createTempFile("TestCopyListingOut", null); + outFile.deleteOnExit(); + List srcs = new ArrayList(); + srcs.add(new Path(inFile.toURI())); + + Exception expectedEx = new IOException("boom"); + SequenceFile.Writer writer = mock(SequenceFile.Writer.class); + doThrow(expectedEx).when(writer).close(); + + SimpleCopyListing listing = new SimpleCopyListing(getConf(), CREDENTIALS); + DistCpOptions options = new DistCpOptions(srcs, new Path(outFile.toURI())); + Exception actualEx = null; + try { + listing.doBuildListing(writer, options); + } catch (Exception e) { + actualEx = e; + } + Assert.assertNotNull("close writer didn't fail", actualEx); + Assert.assertEquals(expectedEx, actualEx); + } } diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestRetriableFileCopyCommand.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestRetriableFileCopyCommand.java new file mode 100644 index 00000000000..c5ec513bec5 --- /dev/null +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestRetriableFileCopyCommand.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.tools.mapred; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapreduce.*; +import org.junit.Test; +import static org.junit.Assert.*; +import static org.mockito.Mockito.*; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; + +public class TestRetriableFileCopyCommand { + @SuppressWarnings("rawtypes") + @Test + public void testFailOnCloseError() throws Exception { + Mapper.Context context = mock(Mapper.Context.class); + doReturn(new Configuration()).when(context).getConfiguration(); + + Exception expectedEx = new IOException("boom"); + OutputStream out = mock(OutputStream.class); + doThrow(expectedEx).when(out).close(); + + File f = File.createTempFile(this.getClass().getSimpleName(), null); + f.deleteOnExit(); + FileStatus stat = + new FileStatus(1L, false, 1, 1024, 0, new Path(f.toURI())); + + Exception actualEx = null; + try { + new RetriableFileCopyCommand("testFailOnCloseError") + .copyBytes(stat, out, 512, context); + } catch (Exception e) { + actualEx = e; + } + assertNotNull("close didn't fail", actualEx); + assertEquals(expectedEx, actualEx); + } +} From 419783d99294cc153fe819d69fa28f6b6e556ba9 Mon Sep 17 00:00:00 2001 From: Jason Darrell Lowe Date: Thu, 5 Dec 2013 16:00:40 +0000 Subject: [PATCH 21/32] HADOOP-10081. Client.setupIOStreams can leak socket resources on exception or error. Contributed by Tsuyoshi OZAWA git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1548179 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../src/main/java/org/apache/hadoop/ipc/Client.java | 1 + 2 files changed, 4 insertions(+) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index b8576d7e759..98275210f3a 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -529,6 +529,9 @@ Release 2.3.0 - UNRELEASED HDFS-5560. Trash configuration log statements prints incorrect units. (Josh Elser via Andrew Wang) + HADOOP-10081. Client.setupIOStreams can leak socket resources on exception + or error (Tsuyoshi OZAWA via jlowe) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java index 764c7dca5ca..e5f23cc7ced 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java @@ -1158,6 +1158,7 @@ public class Client { // cleanup calls cleanupCalls(); } + closeConnection(); if (LOG.isDebugEnabled()) LOG.debug(getName() + ": closed"); } From 13c5d3eb34a013715b0a1974c9b2ee153561b2f0 Mon Sep 17 00:00:00 2001 From: Daryn Sharp Date: Thu, 5 Dec 2013 16:26:43 +0000 Subject: [PATCH 22/32] Move HADOOP-10129 from 0.23.10 to 0.23.11 in CHANGES.txt git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1548194 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 98275210f3a..f03bbcaf179 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -2339,6 +2339,20 @@ Release 2.0.0-alpha - 05-23-2012 HADOOP-7606. Upgrade Jackson to version 1.7.1 to match the version required by Jersey (Alejandro Abdelnur via atm) +Release 0.23.11 - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + + HADOOP-10129. Distcp may succeed when it fails (daryn) + Release 0.23.10 - UNRELEASED INCOMPATIBLE CHANGES @@ -2360,8 +2374,6 @@ Release 0.23.10 - UNRELEASED BUG FIXES - HADOOP-10129. Distcp may succeed when it fails (daryn) - Release 0.23.9 - 2013-07-08 INCOMPATIBLE CHANGES From 0f2ce45aeb0ea391b5f2b357afc4493cf7fa9bc3 Mon Sep 17 00:00:00 2001 From: Jason Darrell Lowe Date: Thu, 5 Dec 2013 16:30:12 +0000 Subject: [PATCH 23/32] MAPREDUCE-5409. MRAppMaster throws InvalidStateTransitonException: Invalid event: TA_TOO_MANY_FETCH_FAILURE at KILLED for TaskAttemptImpl. Contributed by Gera Shegalov git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1548197 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 4 ++ .../v2/app/job/impl/TaskAttemptImpl.java | 40 +++++------ .../v2/app/job/impl/TestTaskAttempt.java | 68 +++++++++++++++++++ 3 files changed, 89 insertions(+), 23 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index fba63583089..42810733e7b 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -280,6 +280,10 @@ Release 2.3.0 - UNRELEASED MAPREDUCE-5451. MR uses LD_LIBRARY_PATH which doesn't mean anything in Windows. (Yingda Chen via cnauroth) + MAPREDUCE-5409. MRAppMaster throws InvalidStateTransitonException: Invalid + event: TA_TOO_MANY_FETCH_FAILURE at KILLED for TaskAttemptImpl (Gera + Shegalov via jlowe) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index ce880f8a699..f3c62a48d5e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -192,6 +192,21 @@ public abstract class TaskAttemptImpl implements DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION = new DiagnosticInformationUpdater(); + private static final EnumSet + FAILED_KILLED_STATE_IGNORED_EVENTS = EnumSet.of( + TaskAttemptEventType.TA_KILL, + TaskAttemptEventType.TA_ASSIGNED, + TaskAttemptEventType.TA_CONTAINER_COMPLETED, + TaskAttemptEventType.TA_UPDATE, + // Container launch events can arrive late + TaskAttemptEventType.TA_CONTAINER_LAUNCHED, + TaskAttemptEventType.TA_CONTAINER_LAUNCH_FAILED, + TaskAttemptEventType.TA_CONTAINER_CLEANED, + TaskAttemptEventType.TA_COMMIT_PENDING, + TaskAttemptEventType.TA_DONE, + TaskAttemptEventType.TA_FAILMSG, + TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE); + private static final StateMachineFactory stateMachineFactory @@ -452,18 +467,7 @@ public abstract class TaskAttemptImpl implements DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION) // Ignore-able events for FAILED state .addTransition(TaskAttemptStateInternal.FAILED, TaskAttemptStateInternal.FAILED, - EnumSet.of(TaskAttemptEventType.TA_KILL, - TaskAttemptEventType.TA_ASSIGNED, - TaskAttemptEventType.TA_CONTAINER_COMPLETED, - TaskAttemptEventType.TA_UPDATE, - // Container launch events can arrive late - TaskAttemptEventType.TA_CONTAINER_LAUNCHED, - TaskAttemptEventType.TA_CONTAINER_LAUNCH_FAILED, - TaskAttemptEventType.TA_CONTAINER_CLEANED, - TaskAttemptEventType.TA_COMMIT_PENDING, - TaskAttemptEventType.TA_DONE, - TaskAttemptEventType.TA_FAILMSG, - TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE)) + FAILED_KILLED_STATE_IGNORED_EVENTS) // Transitions from KILLED state .addTransition(TaskAttemptStateInternal.KILLED, TaskAttemptStateInternal.KILLED, @@ -471,17 +475,7 @@ public abstract class TaskAttemptImpl implements DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION) // Ignore-able events for KILLED state .addTransition(TaskAttemptStateInternal.KILLED, TaskAttemptStateInternal.KILLED, - EnumSet.of(TaskAttemptEventType.TA_KILL, - TaskAttemptEventType.TA_ASSIGNED, - TaskAttemptEventType.TA_CONTAINER_COMPLETED, - TaskAttemptEventType.TA_UPDATE, - // Container launch events can arrive late - TaskAttemptEventType.TA_CONTAINER_LAUNCHED, - TaskAttemptEventType.TA_CONTAINER_LAUNCH_FAILED, - TaskAttemptEventType.TA_CONTAINER_CLEANED, - TaskAttemptEventType.TA_COMMIT_PENDING, - TaskAttemptEventType.TA_DONE, - TaskAttemptEventType.TA_FAILMSG)) + FAILED_KILLED_STATE_IGNORED_EVENTS) // create the topology tables .installTopology(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java index 1129c2fcfc4..5858136d485 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java @@ -550,6 +550,8 @@ public class TestTaskAttempt{ eventHandler.internalError); } + + @Test public void testAppDiognosticEventOnUnassignedTask() throws Exception { ApplicationId appId = ApplicationId.newInstance(1, 2); @@ -599,6 +601,72 @@ public class TestTaskAttempt{ eventHandler.internalError); } + @Test + public void testTooManyFetchFailureAfterKill() throws Exception { + ApplicationId appId = ApplicationId.newInstance(1, 2); + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(appId, 0); + JobId jobId = MRBuilderUtils.newJobId(appId, 1); + TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP); + TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0); + Path jobFile = mock(Path.class); + + MockEventHandler eventHandler = new MockEventHandler(); + TaskAttemptListener taListener = mock(TaskAttemptListener.class); + when(taListener.getAddress()).thenReturn(new InetSocketAddress("localhost", 0)); + + JobConf jobConf = new JobConf(); + jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class); + jobConf.setBoolean("fs.file.impl.disable.cache", true); + jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, ""); + jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, "10"); + + TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class); + when(splits.getLocations()).thenReturn(new String[] {"127.0.0.1"}); + + AppContext appCtx = mock(AppContext.class); + ClusterInfo clusterInfo = mock(ClusterInfo.class); + Resource resource = mock(Resource.class); + when(appCtx.getClusterInfo()).thenReturn(clusterInfo); + when(resource.getMemory()).thenReturn(1024); + + TaskAttemptImpl taImpl = + new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1, + splits, jobConf, taListener, + mock(Token.class), new Credentials(), + new SystemClock(), appCtx); + + NodeId nid = NodeId.newInstance("127.0.0.1", 0); + ContainerId contId = ContainerId.newInstance(appAttemptId, 3); + Container container = mock(Container.class); + when(container.getId()).thenReturn(contId); + when(container.getNodeId()).thenReturn(nid); + when(container.getNodeHttpAddress()).thenReturn("localhost:0"); + + taImpl.handle(new TaskAttemptEvent(attemptId, + TaskAttemptEventType.TA_SCHEDULE)); + taImpl.handle(new TaskAttemptContainerAssignedEvent(attemptId, + container, mock(Map.class))); + taImpl.handle(new TaskAttemptContainerLaunchedEvent(attemptId, 0)); + taImpl.handle(new TaskAttemptEvent(attemptId, + TaskAttemptEventType.TA_DONE)); + taImpl.handle(new TaskAttemptEvent(attemptId, + TaskAttemptEventType.TA_CONTAINER_CLEANED)); + + assertEquals("Task attempt is not in succeeded state", taImpl.getState(), + TaskAttemptState.SUCCEEDED); + taImpl.handle(new TaskAttemptEvent(attemptId, + TaskAttemptEventType.TA_KILL)); + assertEquals("Task attempt is not in KILLED state", taImpl.getState(), + TaskAttemptState.KILLED); + taImpl.handle(new TaskAttemptEvent(attemptId, + TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE)); + assertEquals("Task attempt is not in KILLED state, still", taImpl.getState(), + TaskAttemptState.KILLED); + assertFalse("InternalError occurred trying to handle TA_CONTAINER_CLEANED", + eventHandler.internalError); + } + @Test public void testAppDiognosticEventOnNewTask() throws Exception { ApplicationId appId = ApplicationId.newInstance(1, 2); From ca9a32c9512cc59b4701879634b348cabbd15ed6 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Eagles Date: Thu, 5 Dec 2013 17:37:13 +0000 Subject: [PATCH 24/32] HADOOP-10058. TestMetricsSystemImpl#testInitFirstVerifyStopInvokedImmediately fails on trunk (Chen He via jeagles) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1548215 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java | 1 + 2 files changed, 4 insertions(+) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index f03bbcaf179..9241783c090 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -462,6 +462,9 @@ Release 2.4.0 - UNRELEASED HADOOP-10129. Distcp may succeed when it fails (daryn) + HADOOP-10058. TestMetricsSystemImpl#testInitFirstVerifyStopInvokedImmediately + fails on trunk (Chen He via jeagles) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java index 3b9861ea931..6d9024e3ab8 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java @@ -85,6 +85,7 @@ public class TestMetricsSystemImpl { } @Test public void testInitFirstVerifyStopInvokedImmediately() throws Exception { + DefaultMetricsSystem.shutdown(); new ConfigBuilder().add("*.period", 8) //.add("test.sink.plugin.urls", getPluginUrlsAsString()) .add("test.sink.test.class", TestSink.class.getName()) From 55e5b0653c34a5f4146ce5a97a5b4a88a976d88a Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Thu, 5 Dec 2013 21:09:30 +0000 Subject: [PATCH 25/32] HDFS-5630. Hook up cache directive and pool usage statistics. (wang) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1548309 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../hadoop/hdfs/protocol/CacheDirective.java | 63 ++++--- .../hdfs/protocol/CacheDirectiveStats.java | 51 ++++-- .../hadoop/hdfs/protocol/CachePoolStats.java | 37 ++-- .../hadoop/hdfs/protocolPB/PBHelper.java | 12 +- .../CacheReplicationMonitor.java | 67 ++++--- .../hdfs/server/namenode/CacheManager.java | 16 +- .../hdfs/server/namenode/CachePool.java | 56 +++++- .../apache/hadoop/hdfs/tools/CacheAdmin.java | 53 ++++-- .../main/proto/ClientNamenodeProtocol.proto | 8 +- .../server/namenode/TestCacheDirectives.java | 167 ++++++++++++++---- .../src/test/resources/testCacheAdminConf.xml | 58 ++++++ 12 files changed, 450 insertions(+), 140 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 379b01af436..e41182b58dd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -231,6 +231,8 @@ Trunk (Unreleased) HDFS-5536. Implement HTTP policy for Namenode and DataNode. (Haohui Mai via jing9) + HDFS-5630. Hook up cache directive and pool usage statistics. (wang) + OPTIMIZATIONS HDFS-5349. DNA_CACHE and DNA_UNCACHE should be by blockId only. (cmccabe) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java index 99024491c9c..b6964cac966 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java @@ -46,7 +46,9 @@ public final class CacheDirective implements IntrusiveCollection.Element { private long bytesNeeded; private long bytesCached; - private long filesAffected; + private long filesNeeded; + private long filesCached; + private Element prev; private Element next; @@ -58,9 +60,6 @@ public final class CacheDirective implements IntrusiveCollection.Element { Preconditions.checkArgument(replication > 0); this.replication = replication; this.expiryTime = expiryTime; - this.bytesNeeded = 0; - this.bytesCached = 0; - this.filesAffected = 0; } public long getId() { @@ -112,7 +111,8 @@ public final class CacheDirective implements IntrusiveCollection.Element { return new CacheDirectiveStats.Builder(). setBytesNeeded(bytesNeeded). setBytesCached(bytesCached). - setFilesAffected(filesAffected). + setFilesNeeded(filesNeeded). + setFilesCached(filesCached). setHasExpired(new Date().getTime() > expiryTime). build(); } @@ -131,7 +131,8 @@ public final class CacheDirective implements IntrusiveCollection.Element { append(", expiryTime: ").append(getExpiryTimeString()). append(", bytesNeeded:").append(bytesNeeded). append(", bytesCached:").append(bytesCached). - append(", filesAffected:").append(filesAffected). + append(", filesNeeded:").append(filesNeeded). + append(", filesCached:").append(filesCached). append(" }"); return builder.toString(); } @@ -152,42 +153,60 @@ public final class CacheDirective implements IntrusiveCollection.Element { return new HashCodeBuilder().append(id).toHashCode(); } + // + // Stats related getters and setters + // + + /** + * Resets the byte and file statistics being tracked by this CacheDirective. + */ + public void resetStatistics() { + bytesNeeded = 0; + bytesCached = 0; + filesNeeded = 0; + filesCached = 0; + } + public long getBytesNeeded() { return bytesNeeded; } - public void clearBytesNeeded() { - this.bytesNeeded = 0; - } - - public void addBytesNeeded(long toAdd) { - this.bytesNeeded += toAdd; + public void addBytesNeeded(long bytes) { + this.bytesNeeded += bytes; + pool.addBytesNeeded(bytes); } public long getBytesCached() { return bytesCached; } - public void clearBytesCached() { - this.bytesCached = 0; + public void addBytesCached(long bytes) { + this.bytesCached += bytes; + pool.addBytesCached(bytes); } - public void addBytesCached(long toAdd) { - this.bytesCached += toAdd; + public long getFilesNeeded() { + return filesNeeded; } - public long getFilesAffected() { - return filesAffected; + public void addFilesNeeded(long files) { + this.filesNeeded += files; + pool.addFilesNeeded(files); } - public void clearFilesAffected() { - this.filesAffected = 0; + public long getFilesCached() { + return filesCached; } - public void incrementFilesAffected() { - this.filesAffected++; + public void addFilesCached(long files) { + this.filesCached += files; + pool.addFilesCached(files); } + // + // IntrusiveCollection.Element implementation + // + @SuppressWarnings("unchecked") @Override // IntrusiveCollection.Element public void insertInternal(IntrusiveCollection list, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveStats.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveStats.java index b1c3ed48687..0fd4ca23bb1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveStats.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveStats.java @@ -29,7 +29,8 @@ public class CacheDirectiveStats { public static class Builder { private long bytesNeeded; private long bytesCached; - private long filesAffected; + private long filesNeeded; + private long filesCached; private boolean hasExpired; /** @@ -38,8 +39,8 @@ public class CacheDirectiveStats { * @return New CacheDirectiveStats. */ public CacheDirectiveStats build() { - return new CacheDirectiveStats(bytesNeeded, bytesCached, filesAffected, - hasExpired); + return new CacheDirectiveStats(bytesNeeded, bytesCached, filesNeeded, + filesCached, hasExpired); } /** @@ -71,13 +72,23 @@ public class CacheDirectiveStats { } /** - * Sets the files affected by this directive. - * - * @param filesAffected The files affected. + * Sets the files needed by this directive. + * @param filesNeeded The number of files needed * @return This builder, for call chaining. */ - public Builder setFilesAffected(long filesAffected) { - this.filesAffected = filesAffected; + public Builder setFilesNeeded(long filesNeeded) { + this.filesNeeded = filesNeeded; + return this; + } + + /** + * Sets the files cached by this directive. + * + * @param filesCached The number of files cached. + * @return This builder, for call chaining. + */ + public Builder setFilesCached(long filesCached) { + this.filesCached = filesCached; return this; } @@ -95,14 +106,16 @@ public class CacheDirectiveStats { private final long bytesNeeded; private final long bytesCached; - private final long filesAffected; + private final long filesNeeded; + private final long filesCached; private final boolean hasExpired; private CacheDirectiveStats(long bytesNeeded, long bytesCached, - long filesAffected, boolean hasExpired) { + long filesNeeded, long filesCached, boolean hasExpired) { this.bytesNeeded = bytesNeeded; this.bytesCached = bytesCached; - this.filesAffected = filesAffected; + this.filesNeeded = filesNeeded; + this.filesCached = filesCached; this.hasExpired = hasExpired; } @@ -121,10 +134,17 @@ public class CacheDirectiveStats { } /** - * @return The files affected. + * @return The number of files needed. */ - public long getFilesAffected() { - return filesAffected; + public long getFilesNeeded() { + return filesNeeded; + } + + /** + * @return The number of files cached. + */ + public long getFilesCached() { + return filesCached; } /** @@ -140,7 +160,8 @@ public class CacheDirectiveStats { builder.append("{"); builder.append("bytesNeeded: ").append(bytesNeeded); builder.append(", ").append("bytesCached: ").append(bytesCached); - builder.append(", ").append("filesAffected: ").append(filesAffected); + builder.append(", ").append("filesNeeded: ").append(filesNeeded); + builder.append(", ").append("filesCached: ").append(filesCached); builder.append(", ").append("hasExpired: ").append(hasExpired); builder.append("}"); return builder.toString(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolStats.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolStats.java index 2235447b6f8..c205c15be00 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolStats.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolStats.java @@ -30,7 +30,8 @@ public class CachePoolStats { public static class Builder { private long bytesNeeded; private long bytesCached; - private long filesAffected; + private long filesNeeded; + private long filesCached; public Builder() { } @@ -45,24 +46,33 @@ public class CachePoolStats { return this; } - public Builder setFilesAffected(long filesAffected) { - this.filesAffected = filesAffected; + public Builder setFilesNeeded(long filesNeeded) { + this.filesNeeded = filesNeeded; + return this; + } + + public Builder setFilesCached(long filesCached) { + this.filesCached = filesCached; return this; } public CachePoolStats build() { - return new CachePoolStats(bytesNeeded, bytesCached, filesAffected); + return new CachePoolStats(bytesNeeded, bytesCached, filesNeeded, + filesCached); } }; private final long bytesNeeded; private final long bytesCached; - private final long filesAffected; + private final long filesNeeded; + private final long filesCached; - private CachePoolStats(long bytesNeeded, long bytesCached, long filesAffected) { + private CachePoolStats(long bytesNeeded, long bytesCached, long filesNeeded, + long filesCached) { this.bytesNeeded = bytesNeeded; this.bytesCached = bytesCached; - this.filesAffected = filesAffected; + this.filesNeeded = filesNeeded; + this.filesCached = filesCached; } public long getBytesNeeded() { @@ -70,18 +80,23 @@ public class CachePoolStats { } public long getBytesCached() { - return bytesNeeded; + return bytesCached; } - public long getFilesAffected() { - return filesAffected; + public long getFilesNeeded() { + return filesNeeded; + } + + public long getFilesCached() { + return filesCached; } public String toString() { return new StringBuilder().append("{"). append("bytesNeeded:").append(bytesNeeded). append(", bytesCached:").append(bytesCached). - append(", filesAffected:").append(filesAffected). + append(", filesNeeded:").append(filesNeeded). + append(", filesCached:").append(filesCached). append("}").toString(); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index 8f340b4bf43..3280feaa671 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -1642,7 +1642,8 @@ public class PBHelper { CacheDirectiveStatsProto.newBuilder(); builder.setBytesNeeded(stats.getBytesNeeded()); builder.setBytesCached(stats.getBytesCached()); - builder.setFilesAffected(stats.getFilesAffected()); + builder.setFilesNeeded(stats.getFilesNeeded()); + builder.setFilesCached(stats.getFilesCached()); builder.setHasExpired(stats.hasExpired()); return builder.build(); } @@ -1651,7 +1652,8 @@ public class PBHelper { CacheDirectiveStats.Builder builder = new CacheDirectiveStats.Builder(); builder.setBytesNeeded(proto.getBytesNeeded()); builder.setBytesCached(proto.getBytesCached()); - builder.setFilesAffected(proto.getFilesAffected()); + builder.setFilesNeeded(proto.getFilesNeeded()); + builder.setFilesCached(proto.getFilesCached()); builder.setHasExpired(proto.getHasExpired()); return builder.build(); } @@ -1711,7 +1713,8 @@ public class PBHelper { CachePoolStatsProto.Builder builder = CachePoolStatsProto.newBuilder(); builder.setBytesNeeded(stats.getBytesNeeded()); builder.setBytesCached(stats.getBytesCached()); - builder.setFilesAffected(stats.getFilesAffected()); + builder.setFilesNeeded(stats.getFilesNeeded()); + builder.setFilesCached(stats.getFilesCached()); return builder.build(); } @@ -1719,7 +1722,8 @@ public class PBHelper { CachePoolStats.Builder builder = new CachePoolStats.Builder(); builder.setBytesNeeded(proto.getBytesNeeded()); builder.setBytesCached(proto.getBytesCached()); - builder.setFilesAffected(proto.getFilesAffected()); + builder.setFilesNeeded(proto.getFilesNeeded()); + builder.setFilesCached(proto.getFilesCached()); return builder.build(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java index d0e35680d8c..a36dc84e741 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.protocol.CacheDirective; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.CachedBlocksList.Type; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; import org.apache.hadoop.hdfs.server.namenode.CacheManager; +import org.apache.hadoop.hdfs.server.namenode.CachePool; import org.apache.hadoop.hdfs.server.namenode.CachedBlock; import org.apache.hadoop.hdfs.server.namenode.FSDirectory; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; @@ -198,6 +199,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable { scannedBlocks = 0; namesystem.writeLock(); try { + resetStatistics(); rescanCacheDirectives(); rescanCachedBlockMap(); blockManager.getDatanodeManager().resetLastCachingDirectiveSentTime(); @@ -206,6 +208,15 @@ public class CacheReplicationMonitor extends Thread implements Closeable { } } + private void resetStatistics() { + for (CachePool pool: cacheManager.getCachePools()) { + pool.resetStatistics(); + } + for (CacheDirective directive: cacheManager.getCacheDirectives()) { + directive.resetStatistics(); + } + } + /** * Scan all CacheDirectives. Use the information to figure out * what cache replication factor each block should have. @@ -213,11 +224,9 @@ public class CacheReplicationMonitor extends Thread implements Closeable { private void rescanCacheDirectives() { FSDirectory fsDir = namesystem.getFSDirectory(); final long now = new Date().getTime(); - for (CacheDirective directive : cacheManager.getEntriesById().values()) { - // Reset the directive - directive.clearBytesNeeded(); - directive.clearBytesCached(); - directive.clearFilesAffected(); + for (CacheDirective directive : cacheManager.getCacheDirectives()) { + // Reset the directive's statistics + directive.resetStatistics(); // Skip processing this entry if it has expired LOG.info("Directive expiry is at " + directive.getExpiryTime()); if (directive.getExpiryTime() > 0 && directive.getExpiryTime() <= now) { @@ -262,26 +271,34 @@ public class CacheReplicationMonitor extends Thread implements Closeable { /** * Apply a CacheDirective to a file. - * - * @param pce The CacheDirective to apply. - * @param file The file. + * + * @param directive The CacheDirective to apply. + * @param file The file. */ - private void rescanFile(CacheDirective pce, INodeFile file) { - pce.incrementFilesAffected(); + private void rescanFile(CacheDirective directive, INodeFile file) { BlockInfo[] blockInfos = file.getBlocks(); - long cachedTotal = 0; + + // Increment the "needed" statistics + directive.addFilesNeeded(1); long neededTotal = 0; + for (BlockInfo blockInfo : blockInfos) { + long neededByBlock = + directive.getReplication() * blockInfo.getNumBytes(); + neededTotal += neededByBlock; + } + directive.addBytesNeeded(neededTotal); + + // TODO: Enforce per-pool quotas + + long cachedTotal = 0; for (BlockInfo blockInfo : blockInfos) { if (!blockInfo.getBlockUCState().equals(BlockUCState.COMPLETE)) { // We don't try to cache blocks that are under construction. continue; } - long neededByBlock = - pce.getReplication() * blockInfo.getNumBytes(); - neededTotal += neededByBlock; Block block = new Block(blockInfo.getBlockId()); CachedBlock ncblock = new CachedBlock(block.getBlockId(), - pce.getReplication(), mark); + directive.getReplication(), mark); CachedBlock ocblock = cachedBlocks.get(ncblock); if (ocblock == null) { cachedBlocks.put(ncblock); @@ -294,26 +311,30 @@ public class CacheReplicationMonitor extends Thread implements Closeable { // both get them added to their bytesCached. List cachedOn = ocblock.getDatanodes(Type.CACHED); - long cachedByBlock = Math.min(cachedOn.size(), pce.getReplication()) * - blockInfo.getNumBytes(); + long cachedByBlock = Math.min(cachedOn.size(), + directive.getReplication()) * blockInfo.getNumBytes(); cachedTotal += cachedByBlock; if (mark != ocblock.getMark()) { // Mark hasn't been set in this scan, so update replication and mark. - ocblock.setReplicationAndMark(pce.getReplication(), mark); + ocblock.setReplicationAndMark(directive.getReplication(), mark); } else { // Mark already set in this scan. Set replication to highest value in // any CacheDirective that covers this file. ocblock.setReplicationAndMark((short)Math.max( - pce.getReplication(), ocblock.getReplication()), mark); + directive.getReplication(), ocblock.getReplication()), mark); } } } - pce.addBytesNeeded(neededTotal); - pce.addBytesCached(cachedTotal); + // Increment the "cached" statistics + directive.addBytesCached(cachedTotal); + if (cachedTotal == neededTotal) { + directive.addFilesCached(1); + } if (LOG.isTraceEnabled()) { - LOG.debug("Directive " + pce.getId() + " is caching " + - file.getFullPathName() + ": " + cachedTotal + "/" + neededTotal); + LOG.trace("Directive " + directive.getId() + " is caching " + + file.getFullPathName() + ": " + cachedTotal + "/" + neededTotal + + " bytes"); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java index a4712b6d104..94c62a9cbc1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java @@ -31,6 +31,7 @@ import java.io.DataOutput; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.Iterator; import java.util.LinkedList; import java.util.List; @@ -238,9 +239,20 @@ public final class CacheManager { return active; } - public TreeMap getEntriesById() { + /** + * @return Unmodifiable view of the collection of CachePools. + */ + public Collection getCachePools() { assert namesystem.hasReadLock(); - return directivesById; + return Collections.unmodifiableCollection(cachePools.values()); + } + + /** + * @return Unmodifiable view of the collection of CacheDirectives. + */ + public Collection getCacheDirectives() { + assert namesystem.hasReadLock(); + return Collections.unmodifiableCollection(directivesById.values()); } @VisibleForTesting diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java index af5a3ae093c..249ea66b1d1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java @@ -70,9 +70,14 @@ public final class CachePool { */ @Nonnull private FsPermission mode; - + private int weight; + private long bytesNeeded; + private long bytesCached; + private long filesNeeded; + private long filesCached; + public final static class DirectiveList extends IntrusiveCollection { private CachePool cachePool; @@ -202,6 +207,48 @@ public final class CachePool { setWeight(weight); } + /** + * Resets statistics related to this CachePool + */ + public void resetStatistics() { + bytesNeeded = 0; + bytesCached = 0; + filesNeeded = 0; + filesCached = 0; + } + + public void addBytesNeeded(long bytes) { + bytesNeeded += bytes; + } + + public void addBytesCached(long bytes) { + bytesCached += bytes; + } + + public void addFilesNeeded(long files) { + filesNeeded += files; + } + + public void addFilesCached(long files) { + filesCached += files; + } + + public long getBytesNeeded() { + return bytesNeeded; + } + + public long getBytesCached() { + return bytesCached; + } + + public long getFilesNeeded() { + return filesNeeded; + } + + public long getFilesCached() { + return filesCached; + } + /** * Get statistics about this CachePool. * @@ -209,9 +256,10 @@ public final class CachePool { */ private CachePoolStats getStats() { return new CachePoolStats.Builder(). - setBytesNeeded(0). - setBytesCached(0). - setFilesAffected(0). + setBytesNeeded(bytesNeeded). + setBytesCached(bytesCached). + setFilesNeeded(filesNeeded). + setFilesCached(filesCached). build(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java index 7bfd90b9522..c6dc09360b4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; import org.apache.hadoop.hdfs.protocol.CacheDirectiveStats; import org.apache.hadoop.hdfs.protocol.CachePoolEntry; import org.apache.hadoop.hdfs.protocol.CachePoolInfo; +import org.apache.hadoop.hdfs.protocol.CachePoolStats; import org.apache.hadoop.hdfs.server.namenode.CachePool; import org.apache.hadoop.hdfs.tools.TableListing.Justification; import org.apache.hadoop.ipc.RemoteException; @@ -477,9 +478,10 @@ public class CacheAdmin extends Configured implements Tool { addField("EXPIRY", Justification.LEFT). addField("PATH", Justification.LEFT); if (printStats) { - tableBuilder.addField("NEEDED", Justification.RIGHT). - addField("CACHED", Justification.RIGHT). - addField("FILES", Justification.RIGHT); + tableBuilder.addField("BYTES_NEEDED", Justification.RIGHT). + addField("BYTES_CACHED", Justification.RIGHT). + addField("FILES_NEEDED", Justification.RIGHT). + addField("FILES_CACHED", Justification.RIGHT); } TableListing tableListing = tableBuilder.build(); @@ -507,7 +509,8 @@ public class CacheAdmin extends Configured implements Tool { if (printStats) { row.add("" + stats.getBytesNeeded()); row.add("" + stats.getBytesCached()); - row.add("" + stats.getFilesAffected()); + row.add("" + stats.getFilesNeeded()); + row.add("" + stats.getFilesCached()); } tableListing.addRow(row.toArray(new String[0])); numEntries++; @@ -769,13 +772,14 @@ public class CacheAdmin extends Configured implements Tool { @Override public String getShortUsage() { - return "[" + getName() + " [name]]\n"; + return "[" + getName() + " [-stats] []]\n"; } @Override public String getLongUsage() { TableListing listing = getOptionDescriptionListing(); - listing.addRow("[name]", "If specified, list only the named cache pool."); + listing.addRow("-stats", "Display additional cache pool statistics."); + listing.addRow("", "If specified, list only the named cache pool."); return getShortUsage() + "\n" + WordUtils.wrap("Display information about one or more cache pools, " + @@ -787,6 +791,7 @@ public class CacheAdmin extends Configured implements Tool { @Override public int run(Configuration conf, List args) throws IOException { String name = StringUtils.popFirstNonOption(args); + final boolean printStats = StringUtils.popOption("-stats", args); if (!args.isEmpty()) { System.err.print("Can't understand arguments: " + Joiner.on(" ").join(args) + "\n"); @@ -794,28 +799,42 @@ public class CacheAdmin extends Configured implements Tool { return 1; } DistributedFileSystem dfs = getDFS(conf); - TableListing listing = new TableListing.Builder(). + TableListing.Builder builder = new TableListing.Builder(). addField("NAME", Justification.LEFT). addField("OWNER", Justification.LEFT). addField("GROUP", Justification.LEFT). addField("MODE", Justification.LEFT). - addField("WEIGHT", Justification.RIGHT). - build(); + addField("WEIGHT", Justification.RIGHT); + if (printStats) { + builder. + addField("BYTES_NEEDED", Justification.RIGHT). + addField("BYTES_CACHED", Justification.RIGHT). + addField("FILES_NEEDED", Justification.RIGHT). + addField("FILES_CACHED", Justification.RIGHT); + } + TableListing listing = builder.build(); int numResults = 0; try { RemoteIterator iter = dfs.listCachePools(); while (iter.hasNext()) { CachePoolEntry entry = iter.next(); CachePoolInfo info = entry.getInfo(); - String[] row = new String[5]; + LinkedList row = new LinkedList(); if (name == null || info.getPoolName().equals(name)) { - row[0] = info.getPoolName(); - row[1] = info.getOwnerName(); - row[2] = info.getGroupName(); - row[3] = info.getMode() != null ? info.getMode().toString() : null; - row[4] = - info.getWeight() != null ? info.getWeight().toString() : null; - listing.addRow(row); + row.add(info.getPoolName()); + row.add(info.getOwnerName()); + row.add(info.getGroupName()); + row.add(info.getMode() != null ? info.getMode().toString() : null); + row.add( + info.getWeight() != null ? info.getWeight().toString() : null); + if (printStats) { + CachePoolStats stats = entry.getStats(); + row.add(Long.toString(stats.getBytesNeeded())); + row.add(Long.toString(stats.getBytesCached())); + row.add(Long.toString(stats.getFilesNeeded())); + row.add(Long.toString(stats.getFilesCached())); + } + listing.addRow(row.toArray(new String[] {})); ++numResults; if (name != null) { break; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto index 8fbd31571db..bdd024973e1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto @@ -379,8 +379,9 @@ message CacheDirectiveInfoExpirationProto { message CacheDirectiveStatsProto { required int64 bytesNeeded = 1; required int64 bytesCached = 2; - required int64 filesAffected = 3; - required bool hasExpired = 4; + required int64 filesNeeded = 3; + required int64 filesCached = 4; + required bool hasExpired = 5; } message AddCacheDirectiveRequestProto { @@ -431,7 +432,8 @@ message CachePoolInfoProto { message CachePoolStatsProto { required int64 bytesNeeded = 1; required int64 bytesCached = 2; - required int64 filesAffected = 3; + required int64 filesNeeded = 3; + required int64 filesCached = 4; } message AddCachePoolRequestProto { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java index 36492669114..eb5f7a0d57f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java @@ -62,6 +62,7 @@ import org.apache.hadoop.hdfs.protocol.CacheDirectiveStats; import org.apache.hadoop.hdfs.protocol.CachePoolEntry; import org.apache.hadoop.hdfs.protocol.CachePoolInfo; import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo.Expiration; +import org.apache.hadoop.hdfs.protocol.CachePoolStats; import org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.CachedBlocksList.Type; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; @@ -623,45 +624,111 @@ public class TestCacheDirectives { }, 500, 60000); } - private static void waitForCachedStats(final DistributedFileSystem dfs, - final long targetFilesAffected, final long targetBytesNeeded, - final long targetBytesCached, - final CacheDirectiveInfo filter, final String infoString) + private static void waitForCacheDirectiveStats(final DistributedFileSystem dfs, + final long targetBytesNeeded, final long targetBytesCached, + final long targetFilesNeeded, final long targetFilesCached, + final CacheDirectiveInfo filter, final String infoString) throws Exception { - LOG.info("Polling listDirectives{" + - ((filter == null) ? "ALL" : filter.toString()) + - " for " + targetFilesAffected + " targetFilesAffected, " + - targetBytesNeeded + " targetBytesNeeded, " + - targetBytesCached + " targetBytesCached"); - GenericTestUtils.waitFor(new Supplier() { - @Override - public Boolean get() { - RemoteIterator iter = null; - CacheDirectiveEntry entry = null; + LOG.info("Polling listCacheDirectives " + + ((filter == null) ? "ALL" : filter.toString()) + " for " + + targetBytesNeeded + " targetBytesNeeded, " + + targetBytesCached + " targetBytesCached, " + + targetFilesNeeded + " targetFilesNeeded, " + + targetFilesCached + " targetFilesCached"); + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + RemoteIterator iter = null; + CacheDirectiveEntry entry = null; + try { + iter = dfs.listCacheDirectives(filter); + entry = iter.next(); + } catch (IOException e) { + fail("got IOException while calling " + + "listCacheDirectives: " + e.getMessage()); + } + Assert.assertNotNull(entry); + CacheDirectiveStats stats = entry.getStats(); + if ((targetBytesNeeded == stats.getBytesNeeded()) && + (targetBytesCached == stats.getBytesCached()) && + (targetFilesNeeded == stats.getFilesNeeded()) && + (targetFilesCached == stats.getFilesCached())) { + return true; + } else { + LOG.info(infoString + ": " + + "filesNeeded: " + + stats.getFilesNeeded() + "/" + targetFilesNeeded + + ", filesCached: " + + stats.getFilesCached() + "/" + targetFilesCached + + ", bytesNeeded: " + + stats.getBytesNeeded() + "/" + targetBytesNeeded + + ", bytesCached: " + + stats.getBytesCached() + "/" + targetBytesCached); + return false; + } + } + }, 500, 60000); + } + + private static void waitForCachePoolStats(final DistributedFileSystem dfs, + final long targetBytesNeeded, final long targetBytesCached, + final long targetFilesNeeded, final long targetFilesCached, + final CachePoolInfo pool, final String infoString) + throws Exception { + LOG.info("Polling listCachePools " + pool.toString() + " for " + + targetBytesNeeded + " targetBytesNeeded, " + + targetBytesCached + " targetBytesCached, " + + targetFilesNeeded + " targetFilesNeeded, " + + targetFilesCached + " targetFilesCached"); + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + RemoteIterator iter = null; + try { + iter = dfs.listCachePools(); + } catch (IOException e) { + fail("got IOException while calling " + + "listCachePools: " + e.getMessage()); + } + while (true) { + CachePoolEntry entry = null; try { - iter = dfs.listCacheDirectives(filter); + if (!iter.hasNext()) { + break; + } entry = iter.next(); } catch (IOException e) { - fail("got IOException while calling " + - "listCacheDirectives: " + e.getMessage()); + fail("got IOException while iterating through " + + "listCachePools: " + e.getMessage()); } - Assert.assertNotNull(entry); - CacheDirectiveStats stats = entry.getStats(); - if ((targetFilesAffected == stats.getFilesAffected()) && - (targetBytesNeeded == stats.getBytesNeeded()) && - (targetBytesCached == stats.getBytesCached())) { + if (entry == null) { + break; + } + if (!entry.getInfo().getPoolName().equals(pool.getPoolName())) { + continue; + } + CachePoolStats stats = entry.getStats(); + if ((targetBytesNeeded == stats.getBytesNeeded()) && + (targetBytesCached == stats.getBytesCached()) && + (targetFilesNeeded == stats.getFilesNeeded()) && + (targetFilesCached == stats.getFilesCached())) { return true; } else { - LOG.info(infoString + ": filesAffected: " + - stats.getFilesAffected() + "/" + targetFilesAffected + - ", bytesNeeded: " + + LOG.info(infoString + ": " + + "filesNeeded: " + + stats.getFilesNeeded() + "/" + targetFilesNeeded + + ", filesCached: " + + stats.getFilesCached() + "/" + targetFilesCached + + ", bytesNeeded: " + stats.getBytesNeeded() + "/" + targetBytesNeeded + - ", bytesCached: " + + ", bytesCached: " + stats.getBytesCached() + "/" + targetBytesCached); return false; } } - }, 500, 60000); + return false; + } + }, 500, 60000); } private static void checkNumCachedReplicas(final DistributedFileSystem dfs, @@ -837,7 +904,8 @@ public class TestCacheDirectives { NameNode namenode = cluster.getNameNode(); // Create the pool final String pool = "friendlyPool"; - dfs.addCachePool(new CachePoolInfo(pool)); + final CachePoolInfo poolInfo = new CachePoolInfo(pool); + dfs.addCachePool(poolInfo); // Create some test files final List paths = new LinkedList(); paths.add(new Path("/foo/bar")); @@ -853,6 +921,7 @@ public class TestCacheDirectives { } waitForCachedBlocks(namenode, 0, 0, "testWaitForCachedReplicasInDirectory:0"); + // cache entire directory long id = dfs.addCacheDirective( new CacheDirectiveInfo.Builder(). @@ -861,14 +930,20 @@ public class TestCacheDirectives { setPool(pool). build()); waitForCachedBlocks(namenode, 4, 8, - "testWaitForCachedReplicasInDirectory:1"); + "testWaitForCachedReplicasInDirectory:1:blocks"); // Verify that listDirectives gives the stats we want. - waitForCachedStats(dfs, 2, - 8 * BLOCK_SIZE, 8 * BLOCK_SIZE, + waitForCacheDirectiveStats(dfs, + 4 * numBlocksPerFile * BLOCK_SIZE, 4 * numBlocksPerFile * BLOCK_SIZE, + 2, 2, new CacheDirectiveInfo.Builder(). setPath(new Path("/foo")). build(), - "testWaitForCachedReplicasInDirectory:2"); + "testWaitForCachedReplicasInDirectory:1:directive"); + waitForCachePoolStats(dfs, + 4 * numBlocksPerFile * BLOCK_SIZE, 4 * numBlocksPerFile * BLOCK_SIZE, + 2, 2, + poolInfo, "testWaitForCachedReplicasInDirectory:1:pool"); + long id2 = dfs.addCacheDirective( new CacheDirectiveInfo.Builder(). setPath(new Path("/foo/bar")). @@ -877,28 +952,42 @@ public class TestCacheDirectives { build()); // wait for an additional 2 cached replicas to come up waitForCachedBlocks(namenode, 4, 10, - "testWaitForCachedReplicasInDirectory:3"); + "testWaitForCachedReplicasInDirectory:2:blocks"); // the directory directive's stats are unchanged - waitForCachedStats(dfs, 2, - 8 * BLOCK_SIZE, 8 * BLOCK_SIZE, + waitForCacheDirectiveStats(dfs, + 4 * numBlocksPerFile * BLOCK_SIZE, 4 * numBlocksPerFile * BLOCK_SIZE, + 2, 2, new CacheDirectiveInfo.Builder(). setPath(new Path("/foo")). build(), - "testWaitForCachedReplicasInDirectory:4"); + "testWaitForCachedReplicasInDirectory:2:directive-1"); // verify /foo/bar's stats - waitForCachedStats(dfs, 1, + waitForCacheDirectiveStats(dfs, 4 * numBlocksPerFile * BLOCK_SIZE, // only 3 because the file only has 3 replicas, not 4 as requested. 3 * numBlocksPerFile * BLOCK_SIZE, + 1, + // only 0 because the file can't be fully cached + 0, new CacheDirectiveInfo.Builder(). setPath(new Path("/foo/bar")). build(), - "testWaitForCachedReplicasInDirectory:5"); + "testWaitForCachedReplicasInDirectory:2:directive-2"); + waitForCachePoolStats(dfs, + (4+4) * numBlocksPerFile * BLOCK_SIZE, + (4+3) * numBlocksPerFile * BLOCK_SIZE, + 3, 2, + poolInfo, "testWaitForCachedReplicasInDirectory:2:pool"); + // remove and watch numCached go to 0 dfs.removeCacheDirective(id); dfs.removeCacheDirective(id2); waitForCachedBlocks(namenode, 0, 0, - "testWaitForCachedReplicasInDirectory:6"); + "testWaitForCachedReplicasInDirectory:3:blocks"); + waitForCachePoolStats(dfs, + 0, 0, + 0, 0, + poolInfo, "testWaitForCachedReplicasInDirectory:3:pool"); } finally { cluster.shutdown(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml index 20584927d52..c793bf964ae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml @@ -399,5 +399,63 @@ + + Testing listing cache pool statistics + + -addPool foo -owner bob -group bob -mode 0664 + -addPool bar -owner alice -group alicegroup -mode 0755 + -listPools -stats + + + -removePool foo + -removePool bar + + + + SubstringComparator + Found 2 results. + + + SubstringComparator + bar alice alicegroup rwxr-xr-x 100 0 0 0 0 + + + SubstringComparator + foo bob bob rw-rw-r-- 100 0 0 0 0 + + + + + + Testing listing cache directive statistics + + -addPool pool1 + -addDirective -path /foo -pool pool1 -ttl 2d + -addDirective -path /bar -pool pool1 -ttl 24h + -addDirective -path /baz -replication 2 -pool pool1 -ttl 60m + -listDirectives -pool pool1 -stats + + + -removePool pool1 + + + + SubstringComparator + Found 3 entries + + + SubstringComparator + /foo 0 0 0 0 + + + SubstringComparator + /bar 0 0 0 0 + + + SubstringComparator + /baz 0 0 0 0 + + + From a5fce4fa716707f99bd297f5e37a6869d033dd0c Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Thu, 5 Dec 2013 22:14:29 +0000 Subject: [PATCH 26/32] YARN-1447. Common PB type definitions for container resizing. (Wangda Tan via Sandy Ryza) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1548318 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../src/main/proto/yarn_protos.proto | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 25b712f0866..93dd7c974c3 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -43,6 +43,9 @@ Release 2.4.0 - UNRELEASED YARN-1392. Allow sophisticated app-to-queue placement policies in the Fair Scheduler (Sandy Ryza) + YARN-1447. Common PB type definitions for container resizing. (Wangda Tan + via Sandy Ryza) + IMPROVEMENTS YARN-7. Support CPU resource for DistributedShell. (Junping Du via llu) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 9d4d59e52a9..9c1c4c61e16 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -311,6 +311,22 @@ enum ContainerExitStatusProto { DISKS_FAILED = -101; } +message ContainerResourceIncreaseRequestProto { + optional ContainerIdProto container_id = 1; + optional ResourceProto capability = 2; +} + +message ContainerResourceIncreaseProto { + optional ContainerIdProto container_id = 1; + optional ResourceProto capability = 2; + optional hadoop.common.TokenProto container_token = 3; +} + +message ContainerResourceDecreaseProto { + optional ContainerIdProto container_id = 1; + optional ResourceProto capability = 2; +} + //////////////////////////////////////////////////////////////////////// ////// From common////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////// From 1cdeb83167cdb05db9077d65a9a8440b2aca9bfe Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Thu, 5 Dec 2013 22:16:48 +0000 Subject: [PATCH 27/32] YARN-1447: Forgot to add new files git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1548320 13f79535-47bb-0310-9956-ffa450edef68 --- .../records/ContainerResourceDecrease.java | 78 ++++++++ .../records/ContainerResourceIncrease.java | 84 +++++++++ .../ContainerResourceIncreaseRequest.java | 80 ++++++++ .../pb/ContainerResourceDecreasePBImpl.java | 136 ++++++++++++++ .../pb/ContainerResourceIncreasePBImpl.java | 171 ++++++++++++++++++ ...ontainerResourceIncreaseRequestPBImpl.java | 141 +++++++++++++++ .../api/TestContainerResourceDecrease.java | 66 +++++++ .../api/TestContainerResourceIncrease.java | 74 ++++++++ .../TestContainerResourceIncreaseRequest.java | 68 +++++++ 9 files changed, 898 insertions(+) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerResourceDecrease.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerResourceIncrease.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerResourceIncreaseRequest.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerResourceDecreasePBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerResourceIncreasePBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerResourceIncreaseRequestPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceDecrease.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceIncrease.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceIncreaseRequest.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerResourceDecrease.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerResourceDecrease.java new file mode 100644 index 00000000000..d766d922d7d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerResourceDecrease.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.yarn.util.Records; + +/** + * Used by Application Master to ask Node Manager reduce size of a specified + * container + */ +public abstract class ContainerResourceDecrease { + @Public + public static ContainerResourceDecrease newInstance( + ContainerId existingContainerId, Resource targetCapability) { + ContainerResourceDecrease context = Records + .newRecord(ContainerResourceDecrease.class); + context.setContainerId(existingContainerId); + context.setCapability(targetCapability); + return context; + } + + @Public + public abstract ContainerId getContainerId(); + + @Public + public abstract void setContainerId(ContainerId containerId); + + @Public + public abstract Resource getCapability(); + + @Public + public abstract void setCapability(Resource capability); + + @Override + public int hashCode() { + return getCapability().hashCode() + getContainerId().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other instanceof ContainerResourceDecrease) { + ContainerResourceDecrease ctx = (ContainerResourceDecrease)other; + + if (getContainerId() == null && ctx.getContainerId() != null) { + return false; + } else if (!getContainerId().equals(ctx.getContainerId())) { + return false; + } + + if (getCapability() == null && ctx.getCapability() != null) { + return false; + } else if (!getCapability().equals(ctx.getCapability())) { + return false; + } + + return true; + } else { + return false; + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerResourceIncrease.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerResourceIncrease.java new file mode 100644 index 00000000000..f4c15605cb8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerResourceIncrease.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.yarn.util.Records; + +/** + * Represent a new increased container accepted by Resource Manager + */ +public abstract class ContainerResourceIncrease { + @Public + public static ContainerResourceIncrease newInstance( + ContainerId existingContainerId, Resource targetCapability, Token token) { + ContainerResourceIncrease context = Records + .newRecord(ContainerResourceIncrease.class); + context.setContainerId(existingContainerId); + context.setCapability(targetCapability); + context.setContainerToken(token); + return context; + } + + @Public + public abstract ContainerId getContainerId(); + + @Public + public abstract void setContainerId(ContainerId containerId); + + @Public + public abstract Resource getCapability(); + + @Public + public abstract void setCapability(Resource capability); + + @Public + public abstract Token getContainerToken(); + + @Public + public abstract void setContainerToken(Token token); + + @Override + public int hashCode() { + return getCapability().hashCode() + getContainerId().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other instanceof ContainerResourceIncrease) { + ContainerResourceIncrease ctx = (ContainerResourceIncrease)other; + + if (getContainerId() == null && ctx.getContainerId() != null) { + return false; + } else if (!getContainerId().equals(ctx.getContainerId())) { + return false; + } + + if (getCapability() == null && ctx.getCapability() != null) { + return false; + } else if (!getCapability().equals(ctx.getCapability())) { + return false; + } + + return true; + } else { + return false; + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerResourceIncreaseRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerResourceIncreaseRequest.java new file mode 100644 index 00000000000..9e3b64044cf --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerResourceIncreaseRequest.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.yarn.util.Records; + +/** + * Used by Application Master, send a container resource increase request to + * Resource Manager + */ +@Public +public abstract class ContainerResourceIncreaseRequest { + @Public + public static ContainerResourceIncreaseRequest newInstance( + ContainerId existingContainerId, Resource targetCapability) { + ContainerResourceIncreaseRequest context = Records + .newRecord(ContainerResourceIncreaseRequest.class); + context.setContainerId(existingContainerId); + context.setCapability(targetCapability); + return context; + } + + @Public + public abstract ContainerId getContainerId(); + + @Public + public abstract void setContainerId(ContainerId containerId); + + @Public + public abstract Resource getCapability(); + + @Public + public abstract void setCapability(Resource capability); + + @Override + public int hashCode() { + return getCapability().hashCode() + getContainerId().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other instanceof ContainerResourceIncreaseRequest) { + ContainerResourceIncreaseRequest ctx = + (ContainerResourceIncreaseRequest) other; + + if (getContainerId() == null && ctx.getContainerId() != null) { + return false; + } else if (!getContainerId().equals(ctx.getContainerId())) { + return false; + } + + if (getCapability() == null && ctx.getCapability() != null) { + return false; + } else if (!getCapability().equals(ctx.getCapability())) { + return false; + } + + return true; + } else { + return false; + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerResourceDecreasePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerResourceDecreasePBImpl.java new file mode 100644 index 00000000000..1834132f7ab --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerResourceDecreasePBImpl.java @@ -0,0 +1,136 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl.pb; + +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerResourceDecrease; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceDecreaseProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceDecreaseProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; + +public class ContainerResourceDecreasePBImpl extends ContainerResourceDecrease { + ContainerResourceDecreaseProto proto = ContainerResourceDecreaseProto + .getDefaultInstance(); + ContainerResourceDecreaseProto.Builder builder = null; + boolean viaProto = false; + + private ContainerId existingContainerId = null; + private Resource targetCapability = null; + + public ContainerResourceDecreasePBImpl() { + builder = ContainerResourceDecreaseProto.newBuilder(); + } + + public ContainerResourceDecreasePBImpl(ContainerResourceDecreaseProto proto) { + this.proto = proto; + viaProto = true; + } + + public ContainerResourceDecreaseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public ContainerId getContainerId() { + ContainerResourceDecreaseProtoOrBuilder p = viaProto ? proto : builder; + if (this.existingContainerId != null) { + return this.existingContainerId; + } + if (p.hasContainerId()) { + this.existingContainerId = convertFromProtoFormat(p.getContainerId()); + } + return this.existingContainerId; + } + + @Override + public void setContainerId(ContainerId existingContainerId) { + maybeInitBuilder(); + if (existingContainerId == null) { + builder.clearContainerId(); + } + this.existingContainerId = existingContainerId; + } + + @Override + public Resource getCapability() { + ContainerResourceDecreaseProtoOrBuilder p = viaProto ? proto : builder; + if (this.targetCapability != null) { + return this.targetCapability; + } + if (p.hasCapability()) { + this.targetCapability = convertFromProtoFormat(p.getCapability()); + } + return this.targetCapability; + } + + @Override + public void setCapability(Resource targetCapability) { + maybeInitBuilder(); + if (targetCapability == null) { + builder.clearCapability(); + } + this.targetCapability = targetCapability; + } + + private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) { + return new ContainerIdPBImpl(p); + } + + private ContainerIdProto convertToProtoFormat(ContainerId t) { + return ((ContainerIdPBImpl) t).getProto(); + } + + private Resource convertFromProtoFormat(ResourceProto p) { + return new ResourcePBImpl(p); + } + + private ResourceProto convertToProtoFormat(Resource t) { + return ((ResourcePBImpl) t).getProto(); + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = ContainerResourceDecreaseProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToBuilder() { + if (this.existingContainerId != null) { + builder.setContainerId(convertToProtoFormat(this.existingContainerId)); + } + if (this.targetCapability != null) { + builder.setCapability(convertToProtoFormat(this.targetCapability)); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerResourceIncreasePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerResourceIncreasePBImpl.java new file mode 100644 index 00000000000..4e4f3a7f703 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerResourceIncreasePBImpl.java @@ -0,0 +1,171 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl.pb; + +import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerResourceIncrease; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.Token; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceIncreaseProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceIncreaseProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; + +public class ContainerResourceIncreasePBImpl extends ContainerResourceIncrease { + ContainerResourceIncreaseProto proto = ContainerResourceIncreaseProto + .getDefaultInstance(); + ContainerResourceIncreaseProto.Builder builder = null; + boolean viaProto = false; + + private ContainerId existingContainerId = null; + private Resource targetCapability = null; + private Token token = null; + + public ContainerResourceIncreasePBImpl() { + builder = ContainerResourceIncreaseProto.newBuilder(); + } + + public ContainerResourceIncreasePBImpl(ContainerResourceIncreaseProto proto) { + this.proto = proto; + viaProto = true; + } + + public ContainerResourceIncreaseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public ContainerId getContainerId() { + ContainerResourceIncreaseProtoOrBuilder p = viaProto ? proto : builder; + if (this.existingContainerId != null) { + return this.existingContainerId; + } + if (p.hasContainerId()) { + this.existingContainerId = convertFromProtoFormat(p.getContainerId()); + } + return this.existingContainerId; + } + + @Override + public void setContainerId(ContainerId existingContainerId) { + maybeInitBuilder(); + if (existingContainerId == null) { + builder.clearContainerId(); + } + this.existingContainerId = existingContainerId; + } + + @Override + public Resource getCapability() { + ContainerResourceIncreaseProtoOrBuilder p = viaProto ? proto : builder; + if (this.targetCapability != null) { + return this.targetCapability; + } + if (p.hasCapability()) { + this.targetCapability = convertFromProtoFormat(p.getCapability()); + } + return this.targetCapability; + } + + @Override + public void setCapability(Resource targetCapability) { + maybeInitBuilder(); + if (targetCapability == null) { + builder.clearCapability(); + } + this.targetCapability = targetCapability; + } + + @Override + public Token getContainerToken() { + ContainerResourceIncreaseProtoOrBuilder p = viaProto ? proto : builder; + if (this.token != null) { + return this.token; + } + if (p.hasContainerToken()) { + this.token = convertFromProtoFormat(p.getContainerToken()); + } + return this.token; + } + + @Override + public void setContainerToken(Token token) { + maybeInitBuilder(); + if (token == null) { + builder.clearContainerToken(); + } + this.token = token; + } + + private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) { + return new ContainerIdPBImpl(p); + } + + private ContainerIdProto convertToProtoFormat(ContainerId t) { + return ((ContainerIdPBImpl) t).getProto(); + } + + private Resource convertFromProtoFormat(ResourceProto p) { + return new ResourcePBImpl(p); + } + + private ResourceProto convertToProtoFormat(Resource t) { + return ((ResourcePBImpl) t).getProto(); + } + + private Token convertFromProtoFormat(TokenProto p) { + return new TokenPBImpl(p); + } + + private TokenProto convertToProtoFormat(Token t) { + return ((TokenPBImpl) t).getProto(); + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = ContainerResourceIncreaseProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToBuilder() { + if (this.existingContainerId != null) { + builder.setContainerId(convertToProtoFormat(this.existingContainerId)); + } + if (this.targetCapability != null) { + builder.setCapability(convertToProtoFormat(this.targetCapability)); + } + if (this.token != null) { + builder.setContainerToken(convertToProtoFormat(this.token)); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerResourceIncreaseRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerResourceIncreaseRequestPBImpl.java new file mode 100644 index 00000000000..f5ebf6c0cfe --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerResourceIncreaseRequestPBImpl.java @@ -0,0 +1,141 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl.pb; + +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerResourceIncreaseRequest; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceIncreaseRequestProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceIncreaseRequestProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; + + +public class ContainerResourceIncreaseRequestPBImpl extends + ContainerResourceIncreaseRequest { + ContainerResourceIncreaseRequestProto proto = + ContainerResourceIncreaseRequestProto.getDefaultInstance(); + ContainerResourceIncreaseRequestProto.Builder builder = null; + boolean viaProto = false; + + private ContainerId existingContainerId = null; + private Resource targetCapability = null; + + public ContainerResourceIncreaseRequestPBImpl() { + builder = ContainerResourceIncreaseRequestProto.newBuilder(); + } + + public ContainerResourceIncreaseRequestPBImpl( + ContainerResourceIncreaseRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public ContainerResourceIncreaseRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public ContainerId getContainerId() { + ContainerResourceIncreaseRequestProtoOrBuilder p = viaProto ? proto + : builder; + if (this.existingContainerId != null) { + return this.existingContainerId; + } + if (p.hasContainerId()) { + this.existingContainerId = convertFromProtoFormat(p.getContainerId()); + } + return this.existingContainerId; + } + + @Override + public void setContainerId(ContainerId existingContainerId) { + maybeInitBuilder(); + if (existingContainerId == null) { + builder.clearContainerId(); + } + this.existingContainerId = existingContainerId; + } + + @Override + public Resource getCapability() { + ContainerResourceIncreaseRequestProtoOrBuilder p = viaProto ? proto + : builder; + if (this.targetCapability != null) { + return this.targetCapability; + } + if (p.hasCapability()) { + this.targetCapability = convertFromProtoFormat(p.getCapability()); + } + return this.targetCapability; + } + + @Override + public void setCapability(Resource targetCapability) { + maybeInitBuilder(); + if (targetCapability == null) { + builder.clearCapability(); + } + this.targetCapability = targetCapability; + } + + private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) { + return new ContainerIdPBImpl(p); + } + + private ContainerIdProto convertToProtoFormat(ContainerId t) { + return ((ContainerIdPBImpl) t).getProto(); + } + + private Resource convertFromProtoFormat(ResourceProto p) { + return new ResourcePBImpl(p); + } + + private ResourceProto convertToProtoFormat(Resource t) { + return ((ResourcePBImpl) t).getProto(); + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = ContainerResourceIncreaseRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToBuilder() { + if (this.existingContainerId != null) { + builder.setContainerId(convertToProtoFormat(this.existingContainerId)); + } + if (this.targetCapability != null) { + builder.setCapability(convertToProtoFormat(this.targetCapability)); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceDecrease.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceDecrease.java new file mode 100644 index 00000000000..b3699f7cf5a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceDecrease.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api; + +import junit.framework.Assert; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerResourceDecrease; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.impl.pb.ContainerResourceDecreasePBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceDecreaseProto; +import org.junit.Test; + +public class TestContainerResourceDecrease { + @Test + public void testResourceDecreaseContext() { + ContainerId containerId = ContainerId + .newInstance(ApplicationAttemptId.newInstance( + ApplicationId.newInstance(1234, 3), 3), 7); + Resource resource = Resource.newInstance(1023, 3); + ContainerResourceDecrease ctx = ContainerResourceDecrease.newInstance( + containerId, resource); + + // get proto and recover to ctx + ContainerResourceDecreaseProto proto = + ((ContainerResourceDecreasePBImpl) ctx).getProto(); + ctx = new ContainerResourceDecreasePBImpl(proto); + + // check values + Assert.assertEquals(ctx.getCapability(), resource); + Assert.assertEquals(ctx.getContainerId(), containerId); + } + + @Test + public void testResourceDecreaseContextWithNull() { + ContainerResourceDecrease ctx = ContainerResourceDecrease.newInstance(null, + null); + + // get proto and recover to ctx; + ContainerResourceDecreaseProto proto = + ((ContainerResourceDecreasePBImpl) ctx).getProto(); + ctx = new ContainerResourceDecreasePBImpl(proto); + + // check values + Assert.assertNull(ctx.getCapability()); + Assert.assertNull(ctx.getContainerId()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceIncrease.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceIncrease.java new file mode 100644 index 00000000000..4faf648eaca --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceIncrease.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api; + +import java.util.Arrays; + +import junit.framework.Assert; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerResourceIncrease; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.Token; +import org.apache.hadoop.yarn.api.records.impl.pb.ContainerResourceIncreasePBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceIncreaseProto; +import org.junit.Test; + +public class TestContainerResourceIncrease { + @Test + public void testResourceIncreaseContext() { + byte[] identifier = new byte[] { 1, 2, 3, 4 }; + Token token = Token.newInstance(identifier, "", "".getBytes(), ""); + ContainerId containerId = ContainerId + .newInstance(ApplicationAttemptId.newInstance( + ApplicationId.newInstance(1234, 3), 3), 7); + Resource resource = Resource.newInstance(1023, 3); + ContainerResourceIncrease ctx = ContainerResourceIncrease.newInstance( + containerId, resource, token); + + // get proto and recover to ctx + ContainerResourceIncreaseProto proto = + ((ContainerResourceIncreasePBImpl) ctx).getProto(); + ctx = new ContainerResourceIncreasePBImpl(proto); + + // check values + Assert.assertEquals(ctx.getCapability(), resource); + Assert.assertEquals(ctx.getContainerId(), containerId); + Assert.assertTrue(Arrays.equals(ctx.getContainerToken().getIdentifier() + .array(), identifier)); + } + + @Test + public void testResourceIncreaseContextWithNull() { + ContainerResourceIncrease ctx = ContainerResourceIncrease.newInstance(null, + null, null); + + // get proto and recover to ctx; + ContainerResourceIncreaseProto proto = + ((ContainerResourceIncreasePBImpl) ctx).getProto(); + ctx = new ContainerResourceIncreasePBImpl(proto); + + // check values + Assert.assertNull(ctx.getContainerToken()); + Assert.assertNull(ctx.getCapability()); + Assert.assertNull(ctx.getContainerId()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceIncreaseRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceIncreaseRequest.java new file mode 100644 index 00000000000..386592ea0b3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceIncreaseRequest.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api; + +import junit.framework.Assert; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerResourceIncreaseRequest; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.impl.pb.ContainerResourceIncreaseRequestPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceIncreaseRequestProto; +import org.junit.Test; + +public class TestContainerResourceIncreaseRequest { + @Test + public void ContainerResourceIncreaseRequest() { + ContainerId containerId = ContainerId + .newInstance(ApplicationAttemptId.newInstance( + ApplicationId.newInstance(1234, 3), 3), 7); + Resource resource = Resource.newInstance(1023, 3); + ContainerResourceIncreaseRequest context = ContainerResourceIncreaseRequest + .newInstance(containerId, resource); + + // to proto and get it back + ContainerResourceIncreaseRequestProto proto = + ((ContainerResourceIncreaseRequestPBImpl) context).getProto(); + ContainerResourceIncreaseRequest contextRecover = + new ContainerResourceIncreaseRequestPBImpl(proto); + + // check value + Assert.assertEquals(contextRecover.getContainerId(), containerId); + Assert.assertEquals(contextRecover.getCapability(), resource); + } + + @Test + public void testResourceChangeContextWithNullField() { + ContainerResourceIncreaseRequest context = ContainerResourceIncreaseRequest + .newInstance(null, null); + + // to proto and get it back + ContainerResourceIncreaseRequestProto proto = + ((ContainerResourceIncreaseRequestPBImpl) context).getProto(); + ContainerResourceIncreaseRequest contextRecover = + new ContainerResourceIncreaseRequestPBImpl(proto); + + // check value + Assert.assertNull(contextRecover.getContainerId()); + Assert.assertNull(contextRecover.getCapability()); + } +} From 368d9769f45bf3d2ac7685574a60be0aa9ee59f9 Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Thu, 5 Dec 2013 23:48:37 +0000 Subject: [PATCH 28/32] YARN-1181. Augment MiniYARNCluster to support HA mode (kasha) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1548330 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 2 + .../hadoop/yarn/server/MiniYARNCluster.java | 196 +++++++++++++----- .../yarn/server/TestMiniYARNClusterForHA.java | 71 +++++++ 3 files changed, 216 insertions(+), 53 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYARNClusterForHA.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 93dd7c974c3..1ff2b8047ad 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -141,6 +141,8 @@ Release 2.4.0 - UNRELEASED YARN-1403. Separate out configuration loading from QueueManager in the Fair Scheduler (Sandy Ryza) + YARN-1181. Augment MiniYARNCluster to support HA mode (Karthik Kambatla) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java index dbb65075ef8..9829e86ab68 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java @@ -25,18 +25,21 @@ import java.net.UnknownHostException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.http.HttpConfig; +import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.Shell.ShellCommandExecutor; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.EventHandler; @@ -87,7 +90,7 @@ public class MiniYARNCluster extends CompositeService { } private NodeManager[] nodeManagers; - private ResourceManager resourceManager; + private ResourceManager[] resourceManagers; private ResourceManagerWrapper resourceManagerWrapper; @@ -103,12 +106,14 @@ public class MiniYARNCluster extends CompositeService { /** * @param testName name of the test - * @param noOfNodeManagers the number of node managers in the cluster + * @param numResourceManagers the number of resource managers in the cluster + * @param numNodeManagers the number of node managers in the cluster * @param numLocalDirs the number of nm-local-dirs per nodemanager * @param numLogDirs the number of nm-log-dirs per nodemanager */ - public MiniYARNCluster(String testName, int noOfNodeManagers, - int numLocalDirs, int numLogDirs) { + public MiniYARNCluster( + String testName, int numResourceManagers, int numNodeManagers, + int numLocalDirs, int numLogDirs) { super(testName.replace("$", "")); this.numLocalDirs = numLocalDirs; this.numLogDirs = numLogDirs; @@ -157,28 +162,103 @@ public class MiniYARNCluster extends CompositeService { this.testWorkDir = targetWorkDir; } - resourceManagerWrapper = new ResourceManagerWrapper(); - addService(resourceManagerWrapper); - nodeManagers = new CustomNodeManager[noOfNodeManagers]; - for(int index = 0; index < noOfNodeManagers; index++) { + resourceManagers = new ResourceManager[numResourceManagers]; + for (int i = 0; i < numResourceManagers; i++) { + resourceManagers[i] = new ResourceManager(); + addService(new ResourceManagerWrapper(i)); + } + nodeManagers = new CustomNodeManager[numNodeManagers]; + for(int index = 0; index < numNodeManagers; index++) { addService(new NodeManagerWrapper(index)); nodeManagers[index] = new CustomNodeManager(); } } - - @Override + + /** + * @param testName name of the test + * @param numNodeManagers the number of node managers in the cluster + * @param numLocalDirs the number of nm-local-dirs per nodemanager + * @param numLogDirs the number of nm-log-dirs per nodemanager + */ + public MiniYARNCluster(String testName, int numNodeManagers, + int numLocalDirs, int numLogDirs) { + this(testName, 1, numNodeManagers, numLocalDirs, numLogDirs); + } + + @Override public void serviceInit(Configuration conf) throws Exception { - super.serviceInit(conf instanceof YarnConfiguration ? conf - : new YarnConfiguration( - conf)); + if (resourceManagers.length > 1) { + conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); + + StringBuilder rmIds = new StringBuilder(); + for (int i = 0; i < resourceManagers.length; i++) { + if (i != 0) { + rmIds.append(","); + } + rmIds.append("rm" + i); + } + conf.set(YarnConfiguration.RM_HA_IDS, rmIds.toString()); + } + super.serviceInit( + conf instanceof YarnConfiguration ? conf : new YarnConfiguration(conf)); } public File getTestWorkDir() { return testWorkDir; } + /** + * In a HA cluster, go through all the RMs and find the Active RM. If none + * of them are active, wait upto 5 seconds for them to transition to Active. + * + * In an non-HA cluster, return the index of the only RM. + * + * @return index of the active RM + */ + @InterfaceAudience.Private + @VisibleForTesting + int getActiveRMIndex() { + if (resourceManagers.length == 1) { + return 0; + } + + int numRetriesForRMBecomingActive = 5; + while (numRetriesForRMBecomingActive-- > 0) { + for (int i = 0; i < resourceManagers.length; i++) { + try { + if (HAServiceProtocol.HAServiceState.ACTIVE == + resourceManagers[i].getRMContext().getRMAdminService() + .getServiceStatus().getState()) { + return i; + } + } catch (IOException e) { + throw new YarnRuntimeException("Couldn't read the status of " + + "a ResourceManger in the HA ensemble.", e); + } + } + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new YarnRuntimeException("Interrupted while waiting for one " + + "of the ResourceManagers to become active"); + } + } + return -1; + } + + /** + * @return the active {@link ResourceManager} of the cluster, + * null if none of them are active. + */ public ResourceManager getResourceManager() { - return this.resourceManager; + int activeRMIndex = getActiveRMIndex(); + return activeRMIndex == -1 + ? null + : this.resourceManagers[getActiveRMIndex()]; + } + + public ResourceManager getResourceManager(int i) { + return this.resourceManagers[i]; } public NodeManager getNodeManager(int i) { @@ -195,8 +275,29 @@ public class MiniYARNCluster extends CompositeService { } private class ResourceManagerWrapper extends AbstractService { - public ResourceManagerWrapper() { - super(ResourceManagerWrapper.class.getName()); + private int index; + + public ResourceManagerWrapper(int i) { + super(ResourceManagerWrapper.class.getName() + "_" + i); + index = i; + } + + private void setNonHARMConfiguration(Configuration conf) { + String hostname = MiniYARNCluster.getHostname(); + conf.set(YarnConfiguration.RM_ADDRESS, hostname + ":0"); + conf.set(YarnConfiguration.RM_ADMIN_ADDRESS, hostname + ":0"); + conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, hostname + ":0"); + conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, hostname + ":0"); + WebAppUtils.setRMWebAppHostnameAndPort(conf, hostname, 0); + } + + private void setHARMConfiguration(Configuration conf) { + String rmId = "rm" + index; + String hostname = MiniYARNCluster.getHostname(); + conf.set(YarnConfiguration.RM_HA_ID, rmId); + for (String confKey : YarnConfiguration.RM_RPC_ADDRESS_CONF_KEYS) { + conf.set(HAUtil.addSuffix(confKey, rmId), hostname + ":0"); + } } @Override @@ -206,22 +307,15 @@ public class MiniYARNCluster extends CompositeService { if (!conf.getBoolean( YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, YarnConfiguration.DEFAULT_YARN_MINICLUSTER_FIXED_PORTS)) { - // pick free random ports. - String hostname = MiniYARNCluster.getHostname(); - conf.set(YarnConfiguration.RM_ADDRESS, hostname + ":0"); - conf.set(YarnConfiguration.RM_ADMIN_ADDRESS, hostname + ":0"); - conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, hostname + ":0"); - conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, hostname + ":0"); - WebAppUtils.setRMWebAppHostnameAndPort(conf, hostname, 0); + if (HAUtil.isHAEnabled(conf)) { + setHARMConfiguration(conf); + } else { + setNonHARMConfiguration(conf); + } } - resourceManager = new ResourceManager() { - @Override - protected void doSecureLogin() throws IOException { - // Don't try to login using keytab in the testcase. - }; - }; - resourceManager.init(conf); - resourceManager.getRMContext().getDispatcher().register(RMAppAttemptEventType.class, + resourceManagers[index].init(conf); + resourceManagers[index].getRMContext().getDispatcher().register + (RMAppAttemptEventType.class, new EventHandler() { public void handle(RMAppAttemptEvent event) { if (event instanceof RMAppAttemptRegistrationEvent) { @@ -239,20 +333,20 @@ public class MiniYARNCluster extends CompositeService { try { new Thread() { public void run() { - resourceManager.start(); - }; + resourceManagers[index].start(); + } }.start(); int waitCount = 0; - while (resourceManager.getServiceState() == STATE.INITED + while (resourceManagers[index].getServiceState() == STATE.INITED && waitCount++ < 60) { LOG.info("Waiting for RM to start..."); Thread.sleep(1500); } - if (resourceManager.getServiceState() != STATE.STARTED) { + if (resourceManagers[index].getServiceState() != STATE.STARTED) { // RM could have failed. throw new IOException( "ResourceManager failed to start. Final state is " - + resourceManager.getServiceState()); + + resourceManagers[index].getServiceState()); } super.serviceStart(); } catch (Throwable t) { @@ -278,9 +372,9 @@ public class MiniYARNCluster extends CompositeService { @Override protected synchronized void serviceStop() throws Exception { - if (resourceManager != null) { + if (resourceManagers[index] != null) { waitForAppMastersToFinish(5000); - resourceManager.stop(); + resourceManagers[index].stop(); } super.serviceStop(); @@ -372,7 +466,7 @@ public class MiniYARNCluster extends CompositeService { new Thread() { public void run() { nodeManagers[index].start(); - }; + } }.start(); int waitCount = 0; while (nodeManagers[index].getServiceState() == STATE.INITED @@ -398,12 +492,12 @@ public class MiniYARNCluster extends CompositeService { super.serviceStop(); } } - + private class CustomNodeManager extends NodeManager { @Override protected void doSecureLogin() throws IOException { // Don't try to login using keytab in the testcase. - }; + } @Override protected NodeStatusUpdater createNodeStatusUpdater(Context context, @@ -412,8 +506,8 @@ public class MiniYARNCluster extends CompositeService { healthChecker, metrics) { @Override protected ResourceTracker getRMClient() { - final ResourceTrackerService rt = resourceManager - .getResourceTrackerService(); + final ResourceTrackerService rt = + getResourceManager().getResourceTrackerService(); final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); @@ -424,8 +518,7 @@ public class MiniYARNCluster extends CompositeService { public NodeHeartbeatResponse nodeHeartbeat( NodeHeartbeatRequest request) throws YarnException, IOException { - NodeHeartbeatResponse response = recordFactory.newRecordInstance( - NodeHeartbeatResponse.class); + NodeHeartbeatResponse response; try { response = rt.nodeHeartbeat(request); } catch (YarnException e) { @@ -440,8 +533,7 @@ public class MiniYARNCluster extends CompositeService { public RegisterNodeManagerResponse registerNodeManager( RegisterNodeManagerRequest request) throws YarnException, IOException { - RegisterNodeManagerResponse response = recordFactory. - newRecordInstance(RegisterNodeManagerResponse.class); + RegisterNodeManagerResponse response; try { response = rt.registerNodeManager(request); } catch (YarnException e) { @@ -452,13 +544,11 @@ public class MiniYARNCluster extends CompositeService { return response; } }; - }; + } @Override - protected void stopRMProxy() { - return; - } + protected void stopRMProxy() { } }; - }; + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYARNClusterForHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYARNClusterForHA.java new file mode 100644 index 00000000000..f62124e5d39 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYARNClusterForHA.java @@ -0,0 +1,71 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ha.HAServiceProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.resourcemanager.AdminService; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.fail; + +public class TestMiniYARNClusterForHA { + MiniYARNCluster cluster; + + @Before + public void setup() throws IOException, InterruptedException { + Configuration conf = new YarnConfiguration(); + + cluster = new MiniYARNCluster(TestMiniYARNClusterForHA.class.getName(), + 2, 1, 1, 1); + cluster.init(conf); + cluster.start(); + + cluster.getResourceManager(0).getRMContext().getRMAdminService() + .transitionToActive(new HAServiceProtocol.StateChangeRequestInfo( + HAServiceProtocol.RequestSource.REQUEST_BY_USER)); + + assertFalse("RM never turned active", -1 == cluster.getActiveRMIndex()); + } + + @Test + public void testClusterWorks() throws YarnException, InterruptedException { + ResourceManager rm = cluster.getResourceManager(0); + GetClusterMetricsRequest req = GetClusterMetricsRequest.newInstance(); + + for (int i = 0; i < 600; i++) { + if (1 == rm.getClientRMService().getClusterMetrics(req) + .getClusterMetrics().getNumNodeManagers()) { + return; + } + Thread.sleep(100); + } + fail("NodeManager never registered with the RM"); + } +} From ef1f556cef43cb1ca59ea42506d2389d02514a3e Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Fri, 6 Dec 2013 00:25:02 +0000 Subject: [PATCH 29/32] YARN-1438. Ensure container diagnostics includes exception from container launch. Contributed by Steve Loughran. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1548336 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../yarn/server/nodemanager/DefaultContainerExecutor.java | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 1ff2b8047ad..2a3ef48dd29 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -294,6 +294,9 @@ Release 2.3.0 - UNRELEASED YARN-1407. RM Web UI and REST APIs should uniformly use YarnApplicationState (Sandy Ryza) + YARN-1438. Ensure container diagnostics includes exception from container + launch. (stevel via acmurthy) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java index 507f7a04931..9e2e1113aed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java @@ -213,7 +213,8 @@ public class DefaultContainerExecutor extends ContainerExecutor { LOG.warn("Exception from container-launch with container ID: " + containerId + " and exit code: " + exitCode , e); logOutput(shExec.getOutput()); - String diagnostics = "Exception from container-launch: \n" + String diagnostics = "Exception from container-launch: " + + e + "\n" + StringUtils.stringifyException(e) + "\n" + shExec.getOutput(); container.handle(new ContainerDiagnosticsUpdateEvent(containerId, diagnostics)); From 6b459506c3bd5e0517f33b1d9626ef0437e3db1d Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Fri, 6 Dec 2013 01:30:22 +0000 Subject: [PATCH 30/32] HDFS-5633. Improve OfflineImageViewer to use less memory. Contributed by Jing Zhao. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1548359 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../FileDistributionVisitor.java | 13 ++++++++++- .../ImageLoaderCurrent.java | 23 ++++++++++++++----- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index e41182b58dd..d44af025ffe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -576,6 +576,8 @@ Release 2.4.0 - UNRELEASED HDFS-5581. NameNodeFsck should use only one instance of BlockPlacementPolicy. (vinay via cmccabe) + HDFS-5633. Improve OfflineImageViewer to use less memory. (jing9) + OPTIMIZATIONS HDFS-5239. Allow FSNamesystem lock fairness to be configurable (daryn) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java index 965d053c053..f293db44d3d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java @@ -100,6 +100,18 @@ class FileDistributionVisitor extends TextWriterImageVisitor { @Override void finish() throws IOException { + output(); + super.finish(); + } + + @Override + void finishAbnormally() throws IOException { + System.out.println("*** Image processing finished abnormally. Ending ***"); + output(); + super.finishAbnormally(); + } + + private void output() throws IOException { // write the distribution into the output file write("Size\tNumFiles\n"); for(int i = 0; i < distribution.length; i++) @@ -109,7 +121,6 @@ class FileDistributionVisitor extends TextWriterImageVisitor { System.out.println("totalBlocks = " + totalBlocks); System.out.println("totalSpace = " + totalSpace); System.out.println("maxFileSize = " + maxFileSize); - super.finish(); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java index 25b128de38e..139b8b80037 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java @@ -129,7 +129,7 @@ class ImageLoaderCurrent implements ImageLoader { -40, -41, -42, -43, -44, -45, -46, -47, -48 }; private int imageVersion = 0; - private final Map subtreeMap = new HashMap(); + private final Map subtreeMap = new HashMap(); private final Map dirNodeMap = new HashMap(); /* (non-Javadoc) @@ -500,11 +500,15 @@ class ImageLoaderCurrent implements ImageLoader { // 1. load dir node id long inodeId = in.readLong(); - String dirName = dirNodeMap.get(inodeId); - String oldValue = subtreeMap.put(inodeId, dirName); - if (oldValue != null) { // the subtree has been visited - return; - } + String dirName = dirNodeMap.remove(inodeId); + Boolean visitedRef = subtreeMap.get(inodeId); + if (visitedRef != null) { + if (visitedRef.booleanValue()) { // the subtree has been visited + return; + } else { // first time to visit + subtreeMap.put(inodeId, true); + } + } // else the dir is not linked by a RefNode, thus cannot be revisited // 2. load possible snapshots processSnapshots(in, v, dirName); @@ -695,6 +699,8 @@ class ImageLoaderCurrent implements ImageLoader { if (numBlocks >= 0) { // File if (supportSnapshot) { + // make sure subtreeMap only contains entry for directory + subtreeMap.remove(inodeId); // process file diffs processFileDiffList(in, v, parentName); if (isSnapshotCopy) { @@ -738,6 +744,11 @@ class ImageLoaderCurrent implements ImageLoader { final boolean firstReferred = in.readBoolean(); if (firstReferred) { + // if a subtree is linked by multiple "parents", the corresponding dir + // must be referred by a reference node. we put the reference node into + // the subtreeMap here and let its value be false. when we later visit + // the subtree for the first time, we change the value to true. + subtreeMap.put(inodeId, false); v.visitEnclosingElement(ImageElement.SNAPSHOT_REF_INODE); processINode(in, v, skipBlocks, parentName, isSnapshotCopy); v.leaveEnclosingElement(); // referred inode From eccd7b20937c753b31f81968b6472f9e02fb1f68 Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Fri, 6 Dec 2013 01:32:27 +0000 Subject: [PATCH 31/32] YARN-546. Allow disabling the Fair Scheduler event log. (Sandy Ryza) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1548360 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 2 + .../fair/FairSchedulerConfiguration.java | 9 +++++ .../scheduler/fair/FairSchedulerEventLog.java | 40 ++++++++++--------- .../fair/TestFairSchedulerEventLog.java | 2 +- 4 files changed, 34 insertions(+), 19 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 2a3ef48dd29..b7ac0c498d1 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -143,6 +143,8 @@ Release 2.4.0 - UNRELEASED YARN-1181. Augment MiniYARNCluster to support HA mode (Karthik Kambatla) + YARN-546. Allow disabling the Fair Scheduler event log (Sandy Ryza) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java index ec45cca158e..e271b053737 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java @@ -53,6 +53,11 @@ public class FairSchedulerConfiguration extends Configuration { public static final String ALLOCATION_FILE = CONF_PREFIX + "allocation.file"; protected static final String DEFAULT_ALLOCATION_FILE = "fair-scheduler.xml"; + + /** Whether to enable the Fair Scheduler event log */ + public static final String EVENT_LOG_ENABLED = CONF_PREFIX + "event-log-enabled"; + public static final boolean DEFAULT_EVENT_LOG_ENABLED = false; + protected static final String EVENT_LOG_DIR = "eventlog.dir"; /** Whether pools can be created that were not specified in the FS configuration file @@ -192,6 +197,10 @@ public class FairSchedulerConfiguration extends Configuration { return getBoolean(SIZE_BASED_WEIGHT, DEFAULT_SIZE_BASED_WEIGHT); } + public boolean isEventLogEnabled() { + return getBoolean(EVENT_LOG_ENABLED, DEFAULT_EVENT_LOG_ENABLED); + } + public String getEventlogDir() { return get(EVENT_LOG_DIR, new File(System.getProperty("hadoop.log.dir", "/tmp/")).getAbsolutePath() + File.separator + "fairscheduler"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java index 3f6b1e8babb..05ba330c896 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java @@ -75,26 +75,30 @@ class FairSchedulerEventLog { private DailyRollingFileAppender appender; boolean init(FairSchedulerConfiguration conf) { - try { - logDir = conf.getEventlogDir(); - File logDirFile = new File(logDir); - if (!logDirFile.exists()) { - if (!logDirFile.mkdirs()) { - throw new IOException( - "Mkdirs failed to create " + logDirFile.toString()); + if (conf.isEventLogEnabled()) { + try { + logDir = conf.getEventlogDir(); + File logDirFile = new File(logDir); + if (!logDirFile.exists()) { + if (!logDirFile.mkdirs()) { + throw new IOException( + "Mkdirs failed to create " + logDirFile.toString()); + } } + String username = System.getProperty("user.name"); + logFile = String.format("%s%shadoop-%s-fairscheduler.log", + logDir, File.separator, username); + logDisabled = false; + PatternLayout layout = new PatternLayout("%d{ISO8601}\t%m%n"); + appender = new DailyRollingFileAppender(layout, logFile, "'.'yyyy-MM-dd"); + appender.activateOptions(); + LOG.info("Initialized fair scheduler event log, logging to " + logFile); + } catch (IOException e) { + LOG.error( + "Failed to initialize fair scheduler event log. Disabling it.", e); + logDisabled = true; } - String username = System.getProperty("user.name"); - logFile = String.format("%s%shadoop-%s-fairscheduler.log", - logDir, File.separator, username); - logDisabled = false; - PatternLayout layout = new PatternLayout("%d{ISO8601}\t%m%n"); - appender = new DailyRollingFileAppender(layout, logFile, "'.'yyyy-MM-dd"); - appender.activateOptions(); - LOG.info("Initialized fair scheduler event log, logging to " + logFile); - } catch (IOException e) { - LOG.error( - "Failed to initialize fair scheduler event log. Disabling it.", e); + } else { logDisabled = true; } return !(logDisabled); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java index db777954ca7..633bca494ad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java @@ -44,7 +44,7 @@ public class TestFairSchedulerEventLog { Configuration conf = new YarnConfiguration(); conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, ResourceScheduler.class); - conf.set("mapred.fairscheduler.eventlog.enabled", "true"); + conf.set("yarn.scheduler.fair.event-log-enabled", "true"); // All tests assume only one assignment per node update conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false"); From 9df84c35d5a228e3ae42e90487f2d1f1264e5eea Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Fri, 6 Dec 2013 01:55:13 +0000 Subject: [PATCH 32/32] HDFS-5590. Block ID and generation stamp may be reused when persistBlocks is set to false. Contributed by Jing Zhao. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1548368 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 -- .../hdfs/server/namenode/FSNamesystem.java | 18 ++++-------------- .../apache/hadoop/hdfs/TestPersistBlocks.java | 4 ---- .../hdfs/server/namenode/TestBackupNode.java | 1 - 5 files changed, 7 insertions(+), 21 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index d44af025ffe..e01ead4d4a0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -786,6 +786,9 @@ Release 2.3.0 - UNRELEASED HDFS-5587. add debug information when NFS fails to start with duplicate user or group names (brandonli) + HDFS-5590. Block ID and generation stamp may be reused when persistBlocks is + set to false. (jing9) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index ea65cbf0fe7..3dc06ba8a39 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -166,8 +166,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final boolean DFS_WEBHDFS_ENABLED_DEFAULT = true; public static final String DFS_PERMISSIONS_ENABLED_KEY = "dfs.permissions.enabled"; public static final boolean DFS_PERMISSIONS_ENABLED_DEFAULT = true; - public static final String DFS_PERSIST_BLOCKS_KEY = "dfs.persist.blocks"; - public static final boolean DFS_PERSIST_BLOCKS_DEFAULT = false; public static final String DFS_PERMISSIONS_SUPERUSERGROUP_KEY = "dfs.permissions.superusergroup"; public static final String DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT = "supergroup"; public static final String DFS_ADMIN = "dfs.cluster.administrators"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 6a74b49fa53..a465e8e7a15 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -79,8 +79,6 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_ENABLED_DEFAU import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERSIST_BLOCKS_DEFAULT; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERSIST_BLOCKS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SUPPORT_APPEND_DEFAULT; @@ -153,6 +151,8 @@ import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry; import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; +import org.apache.hadoop.hdfs.protocol.CachePoolEntry; +import org.apache.hadoop.hdfs.protocol.CachePoolInfo; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; @@ -164,8 +164,6 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; -import org.apache.hadoop.hdfs.protocol.CachePoolEntry; -import org.apache.hadoop.hdfs.protocol.CachePoolInfo; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; @@ -365,7 +363,6 @@ public class FSNamesystem implements Namesystem, FSClusterStats, static final int DEFAULT_MAX_CORRUPT_FILEBLOCKS_RETURNED = 100; static int BLOCK_DELETION_INCREMENT = 1000; private final boolean isPermissionEnabled; - private final boolean persistBlocks; private final UserGroupInformation fsOwner; private final String fsOwnerShortUserName; private final String supergroup; @@ -670,13 +667,10 @@ public class FSNamesystem implements Namesystem, FSClusterStats, LOG.info("supergroup = " + supergroup); LOG.info("isPermissionEnabled = " + isPermissionEnabled); - final boolean persistBlocks = conf.getBoolean(DFS_PERSIST_BLOCKS_KEY, - DFS_PERSIST_BLOCKS_DEFAULT); // block allocation has to be persisted in HA using a shared edits directory // so that the standby has up-to-date namespace information String nameserviceId = DFSUtil.getNamenodeNameServiceId(conf); this.haEnabled = HAUtil.isHAEnabled(conf, nameserviceId); - this.persistBlocks = persistBlocks || (haEnabled && HAUtil.usesSharedEditsDir(conf)); // Sanity check the HA-related config. if (nameserviceId != null) { @@ -2635,9 +2629,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } finally { writeUnlock(); } - if (persistBlocks) { - getEditLog().logSync(); - } + getEditLog().logSync(); // Return located block return makeLocatedBlock(newBlock, targets, offset); @@ -2828,9 +2820,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } finally { writeUnlock(); } - if (persistBlocks) { - getEditLog().logSync(); - } + getEditLog().logSync(); return true; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPersistBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPersistBlocks.java index 4ee6a283819..e12c1cb22c0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPersistBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPersistBlocks.java @@ -97,7 +97,6 @@ public class TestPersistBlocks { conf.setInt( CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0); - conf.setBoolean(DFSConfigKeys.DFS_PERSIST_BLOCKS_KEY, true); MiniDFSCluster cluster = null; long len = 0; @@ -157,7 +156,6 @@ public class TestPersistBlocks { conf.setInt( CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0); - conf.setBoolean(DFSConfigKeys.DFS_PERSIST_BLOCKS_KEY, true); MiniDFSCluster cluster = null; long len = 0; @@ -219,7 +217,6 @@ public class TestPersistBlocks { conf.setInt( CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0); - conf.setBoolean(DFSConfigKeys.DFS_PERSIST_BLOCKS_KEY, true); MiniDFSCluster cluster = null; FSDataOutputStream stream; @@ -269,7 +266,6 @@ public class TestPersistBlocks { conf.setInt( CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0); - conf.setBoolean(DFSConfigKeys.DFS_PERSIST_BLOCKS_KEY, true); MiniDFSCluster cluster = null; FSDataOutputStream stream; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java index dfc3667e322..fc56eb48c24 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java @@ -434,7 +434,6 @@ public class TestBackupNode { public void testCanReadData() throws IOException { Path file1 = new Path("/fileToRead.dat"); Configuration conf = new HdfsConfiguration(); - conf.setBoolean(DFSConfigKeys.DFS_PERSIST_BLOCKS_KEY, true); MiniDFSCluster cluster = null; FileSystem fileSys = null; BackupNode backup = null;