From 5eb618ee1f90ccf901edb5d89be181fad1f67d7f Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Fri, 6 Sep 2013 03:06:41 +0000 Subject: [PATCH 01/21] HDFS-4491. Parallel testing HDFS. Contributed by Andrey Klochkov. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1520479 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/pom.xml | 4 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + hadoop-hdfs-project/hadoop-hdfs/pom.xml | 39 ++++++++++++ .../apache/hadoop/hdfs/HftpFileSystem.java | 12 ++-- .../apache/hadoop/hdfs/HsftpFileSystem.java | 4 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 13 ++-- .../org/apache/hadoop/fs/TestFiRename.java | 5 +- .../hadoop/fs/TestResolveHdfsSymlink.java | 7 ++- .../hadoop/fs/TestUrlStreamHandler.java | 12 ++-- .../fs/loadGenerator/TestLoadGenerator.java | 11 ++-- .../apache/hadoop/hdfs/MiniDFSCluster.java | 29 ++++++--- .../hadoop/hdfs/TestClientReportBadBlock.java | 4 -- .../apache/hadoop/hdfs/TestDFSRollback.java | 3 - .../org/apache/hadoop/hdfs/TestDFSShell.java | 13 ++-- .../apache/hadoop/hdfs/TestDecommission.java | 3 +- .../hdfs/TestDistributedFileSystem.java | 5 ++ .../hadoop/hdfs/TestFSInputChecker.java | 3 +- .../hadoop/hdfs/TestFileAppendRestart.java | 3 +- .../hadoop/hdfs/TestFileCorruption.java | 3 +- .../hadoop/hdfs/TestHDFSServerPorts.java | 20 +++---- .../hadoop/hdfs/TestHftpURLTimeouts.java | 19 +++--- .../hadoop/hdfs/TestMiniDFSCluster.java | 20 ++----- .../apache/hadoop/hdfs/TestPersistBlocks.java | 3 +- .../hadoop/hdfs/qjournal/TestNNWithQJM.java | 3 +- .../hdfs/qjournal/server/TestJournalNode.java | 5 +- .../TestOverReplicatedBlocks.java | 1 - .../TestReplicationPolicy.java | 4 +- .../TestReplicationPolicyWithNodeGroup.java | 59 ++++++++++--------- .../server/datanode/TestBPOfferService.java | 4 +- .../hdfs/server/namenode/TestAllowFormat.java | 28 ++++----- .../hdfs/server/namenode/TestAuditLogs.java | 4 +- .../hdfs/server/namenode/TestBackupNode.java | 2 + .../hdfs/server/namenode/TestCheckpoint.java | 14 +++-- .../hdfs/server/namenode/TestClusterId.java | 3 +- .../hdfs/server/namenode/TestEditLog.java | 7 ++- .../namenode/TestEditLogFileOutputStream.java | 4 +- .../namenode/TestEditLogJournalFailures.java | 4 +- .../server/namenode/TestFSEditLogLoader.java | 4 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 5 +- .../server/namenode/TestNameEditsConfigs.java | 5 +- .../server/namenode/TestNameNodeRecovery.java | 4 +- .../namenode/TestNameNodeResourceChecker.java | 15 ++--- .../hdfs/server/namenode/TestStartup.java | 4 +- .../server/namenode/TestStorageRestore.java | 3 +- .../server/namenode/TestTransferFsImage.java | 4 +- .../namenode/ha/TestBootstrapStandby.java | 4 +- .../ha/TestDFSZKFailoverController.java | 8 +-- .../server/namenode/ha/TestEditLogTailer.java | 4 +- .../namenode/ha/TestFailureToReadEdits.java | 4 +- .../hdfs/server/namenode/ha/TestHAFsck.java | 4 +- .../namenode/ha/TestStandbyCheckpoints.java | 4 +- .../TestOfflineEditsViewer.java | 3 +- .../TestDelimitedImageVisitor.java | 3 +- .../TestOfflineImageViewer.java | 4 +- .../hdfs/util/TestAtomicFileOutputStream.java | 6 +- .../hadoop/hdfs/util/TestMD5FileUtils.java | 6 +- .../hadoop/hdfs/web/TestWebHdfsTimeouts.java | 9 +-- hadoop-project/pom.xml | 22 +------ 58 files changed, 275 insertions(+), 229 deletions(-) diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index af5a7f4ee95..6f7fa0f0259 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -780,9 +780,7 @@ org.apache.maven.plugins maven-surefire-plugin - perthread - ${testsThreadCount} - classes + ${testsThreadCount} -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError -DminiClusterDedicatedDirs=true diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 293de2d515d..2015f2954f4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -267,6 +267,8 @@ Release 2.3.0 - UNRELEASED HDFS-5144. Document time unit to NameNodeMetrics. (Akira Ajisaka via suresh) + HDFS-4491. Parallel testing HDFS. (Andrey Klochkov via cnauroth) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index 59abffa0f96..2160d83bac3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -700,5 +700,44 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> + + parallel-tests + + + + maven-antrun-plugin + + + create-parallel-tests-dirs + test-compile + + + + + + + + + + run + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + ${testsThreadCount} + -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError -DminiClusterDedicatedDirs=true + + ${test.build.data}/${surefire.forkNumber} + ${hadoop.tmp.dir}/${surefire.forkNumber} + + + + + + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java index 9d9cde4dc02..dd5e9c6daa0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java @@ -51,7 +51,7 @@ import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifie import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector; import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.hdfs.tools.DelegationTokenFetcher; -import org.apache.hadoop.hdfs.web.URLUtils; +import org.apache.hadoop.hdfs.web.URLConnectionFactory; import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.net.NetUtils; @@ -86,6 +86,8 @@ public class HftpFileSystem extends FileSystem HttpURLConnection.setFollowRedirects(true); } + URLConnectionFactory connectionFactory = URLConnectionFactory.DEFAULT_CONNECTION_FACTORY; + public static final Text TOKEN_KIND = new Text("HFTP delegation"); protected UserGroupInformation ugi; @@ -331,8 +333,8 @@ public class HftpFileSystem extends FileSystem throws IOException { query = addDelegationTokenParam(query); final URL url = getNamenodeURL(path, query); - final HttpURLConnection connection = - (HttpURLConnection)URLUtils.openConnection(url); + final HttpURLConnection connection; + connection = (HttpURLConnection)connectionFactory.openConnection(url); connection.setRequestMethod("GET"); connection.connect(); return connection; @@ -352,12 +354,14 @@ public class HftpFileSystem extends FileSystem } static class RangeHeaderUrlOpener extends ByteRangeInputStream.URLOpener { + URLConnectionFactory connectionFactory = URLConnectionFactory.DEFAULT_CONNECTION_FACTORY; + RangeHeaderUrlOpener(final URL url) { super(url); } protected HttpURLConnection openConnection() throws IOException { - return (HttpURLConnection)URLUtils.openConnection(url); + return (HttpURLConnection)connectionFactory.openConnection(url); } /** Use HTTP Range header for specifying offset. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java index 438d56e52f0..6a3bdba593b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java @@ -40,7 +40,6 @@ import javax.net.ssl.X509TrustManager; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.web.URLUtils; import org.apache.hadoop.util.Time; /** @@ -154,7 +153,8 @@ public class HsftpFileSystem extends HftpFileSystem { query = addDelegationTokenParam(query); final URL url = new URL("https", nnUri.getHost(), nnUri.getPort(), path + '?' + query); - HttpsURLConnection conn = (HttpsURLConnection)URLUtils.openConnection(url); + HttpsURLConnection conn; + conn = (HttpsURLConnection)connectionFactory.openConnection(url); // bypass hostname verification conn.setHostnameVerifier(new DummyHostnameVerifier()); conn.setRequestMethod("GET"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index 9dbb01b3942..8f1c5895f40 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -119,13 +119,15 @@ public class WebHdfsFileSystem extends FileSystem /** SPNEGO authenticator */ private static final KerberosUgiAuthenticator AUTH = new KerberosUgiAuthenticator(); + /** Default connection factory may be overriden in tests to use smaller timeout values */ + URLConnectionFactory connectionFactory = URLConnectionFactory.DEFAULT_CONNECTION_FACTORY; /** Configures connections for AuthenticatedURL */ - private static final ConnectionConfigurator CONN_CONFIGURATOR = + private final ConnectionConfigurator CONN_CONFIGURATOR = new ConnectionConfigurator() { @Override public HttpURLConnection configure(HttpURLConnection conn) throws IOException { - URLUtils.setTimeouts(conn); + connectionFactory.setTimeouts(conn); return conn; } }; @@ -479,10 +481,9 @@ public class WebHdfsFileSystem extends FileSystem final AuthenticatedURL.Token authToken = new AuthenticatedURL.Token(); conn = new AuthenticatedURL(AUTH, CONN_CONFIGURATOR).openConnection( url, authToken); - URLUtils.setTimeouts(conn); } else { LOG.debug("open URL connection"); - conn = (HttpURLConnection)URLUtils.openConnection(url); + conn = (HttpURLConnection)connectionFactory.openConnection(url); } } catch (AuthenticationException e) { throw new IOException(e); @@ -577,7 +578,7 @@ public class WebHdfsFileSystem extends FileSystem checkRetry = false; //Step 2) Submit another Http request with the URL from the Location header with data. - conn = (HttpURLConnection)URLUtils.openConnection(new URL(redirect)); + conn = (HttpURLConnection)connectionFactory.openConnection(new URL(redirect)); conn.setRequestProperty("Content-Type", MediaType.APPLICATION_OCTET_STREAM); conn.setChunkedStreamingMode(32 << 10); //32kB-chunk connect(); @@ -600,7 +601,7 @@ public class WebHdfsFileSystem extends FileSystem disconnect(); checkRetry = false; - conn = (HttpURLConnection)URLUtils.openConnection(new URL(redirect)); + conn = (HttpURLConnection)connectionFactory.openConnection(new URL(redirect)); connect(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/org/apache/hadoop/fs/TestFiRename.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/org/apache/hadoop/fs/TestFiRename.java index 794c0571944..5ad39304da3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/org/apache/hadoop/fs/TestFiRename.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/org/apache/hadoop/fs/TestFiRename.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.fs; +import java.io.File; import java.io.IOException; import java.util.EnumSet; @@ -27,6 +28,7 @@ import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.test.PathUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -56,8 +58,7 @@ public class TestFiRename { private static String addChild = ""; private static byte[] data = { 0 }; - private static String TEST_ROOT_DIR = - System.getProperty("test.build.data", "/tmp") + "/test"; + private static String TEST_ROOT_DIR = PathUtils.getTestDirName(TestFiRename.class); private static Configuration CONF = new Configuration(); static { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestResolveHdfsSymlink.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestResolveHdfsSymlink.java index c48759e1f5e..cfd4a8d418c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestResolveHdfsSymlink.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestResolveHdfsSymlink.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs; +import java.io.File; import static org.junit.Assert.fail; import java.io.FileNotFoundException; @@ -36,6 +37,7 @@ import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifie import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; +import org.apache.hadoop.test.PathUtils; import org.apache.hadoop.test.GenericTestUtils; import org.junit.AfterClass; import org.junit.Assert; @@ -48,6 +50,7 @@ import org.junit.Test; * underlying file system as Hdfs. */ public class TestResolveHdfsSymlink { + private static File TEST_ROOT_DIR = PathUtils.getTestDir(TestResolveHdfsSymlink.class); private static MiniDFSCluster cluster = null; @BeforeClass @@ -80,12 +83,12 @@ public class TestResolveHdfsSymlink { .getUri()); Path alphaLocalPath = new Path(fcLocal.getDefaultFileSystem().getUri() - .toString(), "/tmp/alpha"); + .toString(), new File(TEST_ROOT_DIR, "alpha").getAbsolutePath()); DFSTestUtil.createFile(FileSystem.getLocal(conf), alphaLocalPath, 16, (short) 1, 2); Path linkTarget = new Path(fcLocal.getDefaultFileSystem().getUri() - .toString(), "/tmp"); + .toString(), TEST_ROOT_DIR.getAbsolutePath()); Path hdfsLink = new Path(fcHdfs.getDefaultFileSystem().getUri().toString(), "/tmp/link"); fcHdfs.createSymlink(linkTarget, hdfsLink, true); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestUrlStreamHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestUrlStreamHandler.java index d28736cffae..845eb6314ef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestUrlStreamHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestUrlStreamHandler.java @@ -31,6 +31,7 @@ import java.net.URL; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.test.PathUtils; import org.junit.Test; /** @@ -38,6 +39,8 @@ import org.junit.Test; */ public class TestUrlStreamHandler { + private static File TEST_ROOT_DIR = PathUtils.getTestDir(TestUrlStreamHandler.class); + /** * Test opening and reading from an InputStream through a hdfs:// URL. *

@@ -111,13 +114,12 @@ public class TestUrlStreamHandler { Configuration conf = new HdfsConfiguration(); // Locate the test temporary directory. - File tmpDir = new File(conf.get("hadoop.tmp.dir")); - if (!tmpDir.exists()) { - if (!tmpDir.mkdirs()) - throw new IOException("Cannot create temporary directory: " + tmpDir); + if (!TEST_ROOT_DIR.exists()) { + if (!TEST_ROOT_DIR.mkdirs()) + throw new IOException("Cannot create temporary directory: " + TEST_ROOT_DIR); } - File tmpFile = new File(tmpDir, "thefile"); + File tmpFile = new File(TEST_ROOT_DIR, "thefile"); URI uri = tmpFile.toURI(); FileSystem fs = FileSystem.get(uri, conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/loadGenerator/TestLoadGenerator.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/loadGenerator/TestLoadGenerator.java index e9e14ce8b11..6c7bac31ea8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/loadGenerator/TestLoadGenerator.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/loadGenerator/TestLoadGenerator.java @@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configured; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.test.PathUtils; import org.apache.hadoop.util.Time; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; @@ -39,8 +40,7 @@ import org.junit.Test; public class TestLoadGenerator extends Configured implements Tool { private static final Configuration CONF = new HdfsConfiguration(); private static final int DEFAULT_BLOCK_SIZE = 10; - private static final String OUT_DIR = - System.getProperty("test.build.data","build/test/data"); + private static final File OUT_DIR = PathUtils.getTestDir(TestLoadGenerator.class); private static final File DIR_STRUCTURE_FILE = new File(OUT_DIR, StructureGenerator.DIR_STRUCTURE_FILE_NAME); private static final File FILE_STRUCTURE_FILE = @@ -65,7 +65,7 @@ public class TestLoadGenerator extends Configured implements Tool { StructureGenerator sg = new StructureGenerator(); String[] args = new String[]{"-maxDepth", "2", "-minWidth", "1", "-maxWidth", "2", "-numOfFiles", "2", - "-avgFileSize", "1", "-outDir", OUT_DIR, "-seed", "1"}; + "-avgFileSize", "1", "-outDir", OUT_DIR.getAbsolutePath(), "-seed", "1"}; final int MAX_DEPTH = 1; final int MIN_WIDTH = 3; @@ -133,8 +133,7 @@ public class TestLoadGenerator extends Configured implements Tool { public void testLoadGenerator() throws Exception { final String TEST_SPACE_ROOT = "/test"; - final String SCRIPT_TEST_DIR = new File(System.getProperty("test.build.data", - "/tmp")).getAbsolutePath(); + final String SCRIPT_TEST_DIR = OUT_DIR.getAbsolutePath(); String script = SCRIPT_TEST_DIR + "/" + "loadgenscript"; String script2 = SCRIPT_TEST_DIR + "/" + "loadgenscript2"; File scriptFile1 = new File(script); @@ -156,7 +155,7 @@ public class TestLoadGenerator extends Configured implements Tool { try { DataGenerator dg = new DataGenerator(); dg.setConf(CONF); - String [] args = new String[] {"-inDir", OUT_DIR, "-root", TEST_SPACE_ROOT}; + String [] args = new String[] {"-inDir", OUT_DIR.getAbsolutePath(), "-root", TEST_SPACE_ROOT}; assertEquals(0, dg.run(args)); final int READ_PROBABILITY = 1; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index 7090d498444..222a9b77ef5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -1407,6 +1407,13 @@ public class MiniDFSCluster { * Shutdown all the nodes in the cluster. */ public void shutdown() { + shutdown(false); + } + + /** + * Shutdown all the nodes in the cluster. + */ + public void shutdown(boolean deleteDfsDir) { LOG.info("Shutting down the Mini HDFS Cluster"); if (checkExitOnShutdown) { if (ExitUtil.terminateCalled()) { @@ -1426,6 +1433,11 @@ public class MiniDFSCluster { nameNode = null; } } + if (deleteDfsDir) { + base_dir.delete(); + } else { + base_dir.deleteOnExit(); + } } /** @@ -2118,7 +2130,7 @@ public class MiniDFSCluster { *

  • /data/data<2*dnIndex + 1>
    • *
  • /data/data<2*dnIndex + 2>
    • * - * + * * @param dnIndex datanode index (starts from 0) * @param dirIndex directory index (0 or 1). Index 0 provides access to the * first storage directory. Index 1 provides access to the second @@ -2149,7 +2161,7 @@ public class MiniDFSCluster { public static String getDNCurrentDir(File storageDir) { return storageDir + "/" + Storage.STORAGE_DIR_CURRENT + "/"; } - + /** * Get directory corresponding to block pool directory in the datanode * @param storageDir the storage directory of a datanode. @@ -2255,7 +2267,7 @@ public class MiniDFSCluster { } return null; } - + /** * Get the block metadata file for a block from a given datanode * @@ -2343,14 +2355,17 @@ public class MiniDFSCluster { } else { if (checkDataNodeAddrConfig) { conf.setIfUnset(DFS_DATANODE_ADDRESS_KEY, "127.0.0.1:0"); - conf.setIfUnset(DFS_DATANODE_HTTP_ADDRESS_KEY, "127.0.0.1:0"); - conf.setIfUnset(DFS_DATANODE_IPC_ADDRESS_KEY, "127.0.0.1:0"); } else { conf.set(DFS_DATANODE_ADDRESS_KEY, "127.0.0.1:0"); - conf.set(DFS_DATANODE_HTTP_ADDRESS_KEY, "127.0.0.1:0"); - conf.set(DFS_DATANODE_IPC_ADDRESS_KEY, "127.0.0.1:0"); } } + if (checkDataNodeAddrConfig) { + conf.setIfUnset(DFS_DATANODE_HTTP_ADDRESS_KEY, "127.0.0.1:0"); + conf.setIfUnset(DFS_DATANODE_IPC_ADDRESS_KEY, "127.0.0.1:0"); + } else { + conf.set(DFS_DATANODE_HTTP_ADDRESS_KEY, "127.0.0.1:0"); + conf.set(DFS_DATANODE_IPC_ADDRESS_KEY, "127.0.0.1:0"); + } } private void addToFile(String p, String address) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientReportBadBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientReportBadBlock.java index 51fab6653f3..74c763d5aa7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientReportBadBlock.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientReportBadBlock.java @@ -71,10 +71,6 @@ public class TestClientReportBadBlock { @Before public void startUpCluster() throws IOException { - if (System.getProperty("test.build.data") == null) { // to allow test to be - // run outside of Ant - System.setProperty("test.build.data", "build/test/data"); - } // disable block scanner conf.setInt(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, -1); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRollback.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRollback.java index e54e2777c44..fbcce3946ba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRollback.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRollback.java @@ -20,9 +20,6 @@ package org.apache.hadoop.hdfs; import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType.DATA_NODE; import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType.NAME_NODE; import static org.junit.Assert.*; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.fail; import java.io.File; import java.io.IOException; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java index e7d82688a9d..42873785f37 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java @@ -44,6 +44,7 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.io.compress.BZip2Codec; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.test.PathUtils; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.ToolRunner; @@ -61,9 +62,7 @@ public class TestDFSShell { private static final Log LOG = LogFactory.getLog(TestDFSShell.class); private static AtomicInteger counter = new AtomicInteger(); - static final String TEST_ROOT_DIR = - new Path(System.getProperty("test.build.data","/tmp")) - .toString().replace(' ', '+'); + static final String TEST_ROOT_DIR = PathUtils.getTestDirName(TestDFSShell.class); static Path writeFile(FileSystem fs, Path f) throws IOException { DataOutputStream out = fs.create(f); @@ -482,12 +481,11 @@ public class TestDFSShell { Configuration dstConf = new HdfsConfiguration(); MiniDFSCluster srcCluster = null; MiniDFSCluster dstCluster = null; - String bak = System.getProperty("test.build.data"); + File bak = new File(PathUtils.getTestDir(getClass()), "dfs_tmp_uri"); + bak.mkdirs(); try{ srcCluster = new MiniDFSCluster.Builder(srcConf).numDataNodes(2).build(); - File nameDir = new File(new File(bak), "dfs_tmp_uri/"); - nameDir.mkdirs(); - System.setProperty("test.build.data", nameDir.toString()); + dstConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, bak.getAbsolutePath()); dstCluster = new MiniDFSCluster.Builder(dstConf).numDataNodes(2).build(); FileSystem srcFs = srcCluster.getFileSystem(); FileSystem dstFs = dstCluster.getFileSystem(); @@ -559,7 +557,6 @@ public class TestDFSShell { ret = ToolRunner.run(shell, argv); assertEquals("default works for rm/rmr", 0, ret); } finally { - System.setProperty("test.build.data", bak); if (null != srcCluster) { srcCluster.shutdown(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java index 05d22961dd0..9c2e038f3f1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; +import org.apache.hadoop.test.PathUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -75,7 +76,7 @@ public class TestDecommission { // Set up the hosts/exclude files. localFileSys = FileSystem.getLocal(conf); Path workingDir = localFileSys.getWorkingDirectory(); - Path dir = new Path(workingDir, System.getProperty("test.build.data", "target/test/data") + "/work-dir/decommission"); + Path dir = new Path(workingDir, PathUtils.getTestDirName(getClass()) + "/work-dir/decommission"); hostsFile = new Path(dir, "hosts"); excludeFile = new Path(dir, "exclude"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java index 9b4f3130d7f..c0093d21aa7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java @@ -25,6 +25,8 @@ import static org.junit.Assert.fail; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; + +import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; @@ -73,6 +75,9 @@ public class TestDistributedFileSystem { HdfsConfiguration conf; if (noXmlDefaults) { conf = new HdfsConfiguration(false); + String namenodeDir = new File(MiniDFSCluster.getBaseDirectory(), "name").getAbsolutePath(); + conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, namenodeDir); + conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, namenodeDir); } else { conf = new HdfsConfiguration(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFSInputChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFSInputChecker.java index 2f2c9a4fd6e..5fc567a2131 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFSInputChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFSInputChecker.java @@ -39,6 +39,7 @@ import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.test.PathUtils; import org.junit.Test; /** @@ -260,7 +261,7 @@ public class TestFSInputChecker { // create a file and verify that checksum corruption results in // a checksum exception on LocalFS - String dir = System.getProperty("test.build.data", "."); + String dir = PathUtils.getTestDirName(getClass()); Path file = new Path(dir + "/corruption-test.dat"); Path crcFile = new Path(dir + "/.corruption-test.dat.crc"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppendRestart.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppendRestart.java index d086c77a9bf..570b19f8d2e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppendRestart.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppendRestart.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.server.namenode.NNStorage; import org.apache.hadoop.hdfs.util.Holder; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.test.PathUtils; import org.junit.Test; /** @@ -145,7 +146,7 @@ public class TestFileAppendRestart { String tarFile = System.getProperty("test.cache.data", "build/test/cache") + "/" + HADOOP_23_BROKEN_APPEND_TGZ; - String testDir = System.getProperty("test.build.data", "build/test/data"); + String testDir = PathUtils.getTestDirName(getClass()); File dfsDir = new File(testDir, "image-with-buggy-append"); if (dfsDir.exists() && !FileUtil.fullyDelete(dfsDir)) { throw new IOException("Could not delete dfs directory '" + dfsDir + "'"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java index 458880af566..6531fe7c050 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; +import org.apache.hadoop.test.PathUtils; import org.apache.log4j.Level; import org.junit.Test; @@ -95,7 +96,7 @@ public class TestFileCorruption { @Test public void testLocalFileCorruption() throws Exception { Configuration conf = new HdfsConfiguration(); - Path file = new Path(System.getProperty("test.build.data"), "corruptFile"); + Path file = new Path(PathUtils.getTestDirName(getClass()), "corruptFile"); FileSystem fs = FileSystem.getLocal(conf); DataOutputStream dos = fs.create(file); dos.writeBytes("original bytes"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSServerPorts.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSServerPorts.java index 036252ddc31..59d1615025d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSServerPorts.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSServerPorts.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hdfs.server.namenode.BackupNode; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.net.DNS; +import org.apache.hadoop.test.PathUtils; import org.junit.Test; /** @@ -53,6 +54,9 @@ public class TestHDFSServerPorts { // reset default 0.0.0.0 addresses in order to avoid IPv6 problem static final String THIS_HOST = getFullHostName() + ":0"; + + private static final File TEST_DATA_DIR = PathUtils.getTestDir(TestHDFSServerPorts.class); + static { DefaultMetricsSystem.setMiniClusterMode(true); } @@ -81,13 +85,6 @@ public class TestHDFSServerPorts { } } - /** - * Get base directory these tests should run in. - */ - private String getTestingDir() { - return System.getProperty("test.build.data", "build/test/data"); - } - public NameNode startNameNode() throws IOException { return startNameNode(false); } @@ -95,8 +92,7 @@ public class TestHDFSServerPorts { * Start the namenode. */ public NameNode startNameNode(boolean withService) throws IOException { - String dataDir = getTestingDir(); - hdfsDir = new File(dataDir, "dfs"); + hdfsDir = new File(TEST_DATA_DIR, "dfs"); if ( hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir) ) { throw new IOException("Could not delete hdfs directory '" + hdfsDir + "'"); } @@ -119,9 +115,8 @@ public class TestHDFSServerPorts { * Start the BackupNode */ public BackupNode startBackupNode(Configuration conf) throws IOException { - String dataDir = getTestingDir(); // Set up testing environment directories - hdfsDir = new File(dataDir, "backupNode"); + hdfsDir = new File(TEST_DATA_DIR, "backupNode"); if ( hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir) ) { throw new IOException("Could not delete hdfs directory '" + hdfsDir + "'"); } @@ -150,8 +145,7 @@ public class TestHDFSServerPorts { */ public DataNode startDataNode(int index, Configuration config) throws IOException { - String dataDir = getTestingDir(); - File dataNodeDir = new File(dataDir, "data-" + index); + File dataNodeDir = new File(TEST_DATA_DIR, "data-" + index); config.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dataNodeDir.getPath()); String[] args = new String[] {}; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java index d9a22c10111..56bd21ef2ff 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java @@ -33,16 +33,11 @@ import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.hdfs.web.URLUtils; -import org.junit.BeforeClass; +import org.apache.hadoop.hdfs.web.URLConnectionFactory; import org.junit.Test; public class TestHftpURLTimeouts { - @BeforeClass - public static void setup() { - URLUtils.SOCKET_TIMEOUT = 5; - } - + @Test public void testHftpSocketTimeout() throws Exception { Configuration conf = new Configuration(); @@ -51,9 +46,11 @@ public class TestHftpURLTimeouts { InetAddress.getByName(null).getHostAddress(), socket.getLocalPort(), null, null, null); - boolean timedout = false; HftpFileSystem fs = (HftpFileSystem)FileSystem.get(uri, conf); + fs.connectionFactory = new URLConnectionFactory(5); + + boolean timedout = false; try { HttpURLConnection conn = fs.openConnection("/", ""); timedout = false; @@ -69,6 +66,7 @@ public class TestHftpURLTimeouts { assertTrue("read timedout", timedout); assertTrue("connect timedout", checkConnectTimeout(fs, false)); } finally { + fs.connectionFactory = URLConnectionFactory.DEFAULT_CONNECTION_FACTORY; fs.close(); } } @@ -84,6 +82,8 @@ public class TestHftpURLTimeouts { boolean timedout = false; HsftpFileSystem fs = (HsftpFileSystem)FileSystem.get(uri, conf); + fs.connectionFactory = new URLConnectionFactory(5); + try { HttpURLConnection conn = null; timedout = false; @@ -100,6 +100,7 @@ public class TestHftpURLTimeouts { assertTrue("ssl read connect timedout", timedout); assertTrue("connect timedout", checkConnectTimeout(fs, true)); } finally { + fs.connectionFactory = URLConnectionFactory.DEFAULT_CONNECTION_FACTORY; fs.close(); } } @@ -121,7 +122,7 @@ public class TestHftpURLTimeouts { // https will get a read timeout due to SSL negotiation, but // a normal http will not, so need to ignore SSL read timeouts // until a connect timeout occurs - if (!(ignoreReadTimeout && message.equals("Read timed out"))) { + if (!(ignoreReadTimeout && "Read timed out".equals(message))) { timedout = true; assertEquals("connect timed out", message); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java index 1400f07e062..84678da2d40 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java @@ -25,8 +25,8 @@ import static org.junit.Assume.assumeTrue; import java.io.File; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.protocol.FSConstants; -import org.junit.After; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.test.PathUtils; import org.junit.Before; import org.junit.Test; @@ -43,20 +43,10 @@ public class TestMiniDFSCluster { private static final String CLUSTER_3 = "cluster3"; private static final String CLUSTER_4 = "cluster4"; private static final String CLUSTER_5 = "cluster5"; - protected String testDataPath; - protected File testDataDir; + protected File testDataPath; @Before public void setUp() { - testDataPath = System.getProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA, - "build/test/data"); - testDataDir = new File(new File(testDataPath).getParentFile(), - "miniclusters"); - - - } - @After - public void tearDown() { - System.setProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA, testDataPath); + testDataPath = new File(PathUtils.getTestDir(getClass()), "miniclusters"); } /** @@ -120,7 +110,7 @@ public class TestMiniDFSCluster { MiniDFSCluster cluster4 = new MiniDFSCluster.Builder(conf).build(); try { DistributedFileSystem dfs = (DistributedFileSystem) cluster4.getFileSystem(); - dfs.setSafeMode(FSConstants.SafeModeAction.SAFEMODE_ENTER); + dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER); cluster4.shutdown(); } finally { while(cluster4.isClusterUp()){ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPersistBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPersistBlocks.java index 5077a6dfdfd..424cc77a19d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPersistBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPersistBlocks.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.test.PathUtils; import org.apache.log4j.Level; import org.junit.Test; @@ -322,7 +323,7 @@ public class TestPersistBlocks { String tarFile = System.getProperty("test.cache.data", "build/test/cache") + "/" + HADOOP_1_0_MULTIBLOCK_TGZ; - String testDir = System.getProperty("test.build.data", "build/test/data"); + String testDir = PathUtils.getTestDirName(getClass()); File dfsDir = new File(testDir, "image-1.0"); if (dfsDir.exists() && !FileUtil.fullyDelete(dfsDir)) { throw new IOException("Could not delete dfs directory '" + dfsDir + "'"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java index cfe463c4356..a7eed468e62 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java @@ -174,6 +174,7 @@ public class TestNNWithQJM { public void testMismatchedNNIsRejected() throws Exception { conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, MiniDFSCluster.getBaseDirectory() + "/TestNNWithQJM/image"); + String defaultEditsDir = conf.get(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY); conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, mjc.getQuorumJournalURI("myjournal").toString()); @@ -187,7 +188,7 @@ public class TestNNWithQJM { // Reformat just the on-disk portion Configuration onDiskOnly = new Configuration(conf); - onDiskOnly.unset(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY); + onDiskOnly.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, defaultEditsDir); NameNode.format(onDiskOnly); // Start the NN - should fail because the JNs are still formatted diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java index e6e140443bf..79ca6ca72bb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java @@ -46,6 +46,7 @@ import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.MetricsAsserts; +import org.apache.hadoop.test.PathUtils; import org.apache.hadoop.util.Shell; import org.junit.After; import org.junit.Before; @@ -61,13 +62,13 @@ public class TestJournalNode { private static final NamespaceInfo FAKE_NSINFO = new NamespaceInfo( 12345, "mycluster", "my-bp", 0L); + private static File TEST_BUILD_DATA = PathUtils.getTestDir(TestJournalNode.class); + private JournalNode jn; private Journal journal; private Configuration conf = new Configuration(); private IPCLoggerChannel ch; private String journalId; - private File TEST_BUILD_DATA = - new File(System.getProperty("test.build.data", "build/test/data")); static { // Avoid an error when we double-initialize JvmMetrics diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java index 79785961c91..972a785f5d6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java @@ -22,7 +22,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import java.io.File; -import java.io.IOException; import java.util.Collection; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java index ba6c3737266..5f15d0e2cb9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java @@ -53,6 +53,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.Namesystem; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.net.Node; +import org.apache.hadoop.test.PathUtils; import org.apache.hadoop.util.Time; import org.apache.log4j.Level; import org.apache.log4j.Logger; @@ -92,8 +93,7 @@ public class TestReplicationPolicy { FileSystem.setDefaultUri(conf, "hdfs://localhost:0"); conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0"); - File baseDir = new File(System.getProperty( - "test.build.data", "build/test/data"), "dfs/"); + File baseDir = PathUtils.getTestDir(TestReplicationPolicy.class); conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, new File(baseDir, "name").getPath()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java index 032c2c08396..c453f198e1d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java @@ -22,7 +22,6 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.io.File; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -41,6 +40,9 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.net.Node; +import org.apache.hadoop.test.PathUtils; +import org.junit.After; +import org.junit.Before; import org.junit.Test; public class TestReplicationPolicyWithNodeGroup { @@ -48,10 +50,10 @@ public class TestReplicationPolicyWithNodeGroup { private static final int NUM_OF_DATANODES = 8; private static final int NUM_OF_DATANODES_BOUNDARY = 6; private static final int NUM_OF_DATANODES_MORE_TARGETS = 12; - private static final Configuration CONF = new HdfsConfiguration(); - private static final NetworkTopology cluster; - private static final NameNode namenode; - private static final BlockPlacementPolicy replicator; + private final Configuration CONF = new HdfsConfiguration(); + private NetworkTopology cluster; + private NameNode namenode; + private BlockPlacementPolicy replicator; private static final String filename = "/dummyfile.txt"; private final static DatanodeDescriptor dataNodes[] = new DatanodeDescriptor[] { @@ -94,27 +96,23 @@ public class TestReplicationPolicyWithNodeGroup { private final static DatanodeDescriptor NODE = new DatanodeDescriptor(DFSTestUtil.getDatanodeDescriptor("9.9.9.9", "/d2/r4/n7")); - static { - try { - FileSystem.setDefaultUri(CONF, "hdfs://localhost:0"); - CONF.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0"); - // Set properties to make HDFS aware of NodeGroup. - CONF.set("dfs.block.replicator.classname", - "org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyWithNodeGroup"); - CONF.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_IMPL_KEY, - "org.apache.hadoop.net.NetworkTopologyWithNodeGroup"); - - File baseDir = new File(System.getProperty( - "test.build.data", "build/test/data"), "dfs/"); - CONF.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, - new File(baseDir, "name").getPath()); - - DFSTestUtil.formatNameNode(CONF); - namenode = new NameNode(CONF); - } catch (IOException e) { - e.printStackTrace(); - throw (RuntimeException)new RuntimeException().initCause(e); - } + @Before + public void setUp() throws Exception { + FileSystem.setDefaultUri(CONF, "hdfs://localhost:0"); + CONF.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0"); + // Set properties to make HDFS aware of NodeGroup. + CONF.set("dfs.block.replicator.classname", + "org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyWithNodeGroup"); + CONF.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_IMPL_KEY, + "org.apache.hadoop.net.NetworkTopologyWithNodeGroup"); + + File baseDir = PathUtils.getTestDir(TestReplicationPolicyWithNodeGroup.class); + + CONF.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, + new File(baseDir, "name").getPath()); + + DFSTestUtil.formatNameNode(CONF); + namenode = new NameNode(CONF); final BlockManager bm = namenode.getNamesystem().getBlockManager(); replicator = bm.getBlockPlacementPolicy(); cluster = bm.getDatanodeManager().getNetworkTopology(); @@ -125,6 +123,11 @@ public class TestReplicationPolicyWithNodeGroup { setupDataNodeCapacity(); } + @After + public void tearDown() throws Exception { + namenode.stop(); + } + private static void setupDataNodeCapacity() { for(int i=0; i fsImageDirs = new ArrayList(); ArrayList editsDirs = new ArrayList(); File filePath = - new File(System.getProperty("test.build.data","/tmp"), "storageDirToCheck"); + new File(PathUtils.getTestDir(getClass()), "storageDirToCheck"); assertTrue("Couldn't create directory storageDirToCheck", filePath.exists() || filePath.mkdirs()); fsImageDirs.add(filePath.toURI()); @@ -1911,9 +1912,11 @@ public class TestCheckpoint { } // Start a new NN with the same host/port. - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0) - .nameNodePort(origPort).nameNodeHttpPort(origHttpPort).format(true) - .build(); + cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(0) + .nameNodePort(origPort) + .nameNodeHttpPort(origHttpPort) + .format(true).build(); try { secondary.doCheckpoint(); @@ -2135,7 +2138,8 @@ public class TestCheckpoint { conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_KEY, 1); try { - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0) + cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(0) .format(true).build(); FileSystem fs = cluster.getFileSystem(); secondary = startSecondaryNameNode(conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java index 4330317d6ff..7c23dd55df5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java @@ -42,6 +42,7 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; +import org.apache.hadoop.test.PathUtils; import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.ExitUtil.ExitException; import org.junit.After; @@ -72,7 +73,7 @@ public class TestClusterId { public void setUp() throws IOException { ExitUtil.disableSystemExit(); - String baseDir = System.getProperty("test.build.data", "build/test/data"); + String baseDir = PathUtils.getTestDirName(getClass()); hdfsDir = new File(baseDir, "dfs/name"); if (hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir)) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java index f83a531b463..1be4a228895 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java @@ -69,6 +69,7 @@ import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.test.PathUtils; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; import org.apache.log4j.Level; @@ -96,9 +97,8 @@ public class TestEditLog { static final int NUM_TRANSACTIONS = 100; static final int NUM_THREADS = 100; - static final File TEST_DIR = new File( - System.getProperty("test.build.data","build/test/data")); - + static final File TEST_DIR = PathUtils.getTestDir(TestEditLog.class); + /** An edits log with 3 edits from 0.20 - the result of * a fresh namesystem followed by hadoop fs -touchz /myfile */ static final byte[] HADOOP20_SOME_EDITS = @@ -569,6 +569,7 @@ public class TestEditLog { fail("should not be able to start"); } catch (IOException e) { // expected + assertNotNull("Cause of exception should be ChecksumException", e.getCause()); assertEquals("Cause of exception should be ChecksumException", ChecksumException.class, e.getCause().getClass()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java index a6e170dee71..e230d5affc6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java @@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue; import java.io.File; import java.io.IOException; +import org.apache.hadoop.test.PathUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; import org.junit.After; @@ -34,8 +35,7 @@ import org.junit.Test; * Test the EditLogFileOutputStream */ public class TestEditLogFileOutputStream { - private final static File TEST_DIR = - new File(System.getProperty("test.build.data", "/tmp")); + private final static File TEST_DIR = PathUtils.getTestDir(TestEditLogFileOutputStream.class); private static final File TEST_EDITS = new File(TEST_DIR, "testEditLogFileOutput.log"); final static int MIN_PREALLOCATION_LENGTH = diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java index 486b17c3925..610a4a2fd4c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.server.namenode.JournalSet.JournalAndStream; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.test.PathUtils; import org.apache.hadoop.util.ExitUtil.ExitException; import org.junit.After; import org.junit.Before; @@ -192,8 +193,7 @@ public class TestEditLogJournalFailures { Configuration conf = new HdfsConfiguration(); String[] nameDirs = new String[4]; for (int i = 0; i < nameDirs.length; i++) { - File nameDir = new File(System.getProperty("test.build.data"), - "name-dir" + i); + File nameDir = new File(PathUtils.getTestDir(getClass()), "name-dir" + i); nameDir.mkdirs(); nameDirs[i] = nameDir.getAbsolutePath(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java index dd637a9212d..4db7e6a7dfa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java @@ -47,6 +47,7 @@ import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.test.PathUtils; import org.apache.log4j.Level; import org.junit.Test; @@ -60,8 +61,7 @@ public class TestFSEditLogLoader { ((Log4JLogger)FSEditLogLoader.LOG).getLogger().setLevel(Level.ALL); } - private static final File TEST_DIR = new File( - System.getProperty("test.build.data","build/test/data")); + private static final File TEST_DIR = PathUtils.getTestDir(TestFSEditLogLoader.class); private static final int NUM_DATA_NODES = 0; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java index 754e56966d3..9aaeb74a1c3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java @@ -91,7 +91,7 @@ import static org.mockito.Mockito.*; */ public class TestFsck { static final String auditLogFile = System.getProperty("test.build.dir", - "build/test") + "/audit.log"; + "build/test") + "/TestFsck-audit.log"; // Pattern for: // allowed=true ugi=name ip=/address cmd=FSCK src=/ dst=null perm=null @@ -159,7 +159,8 @@ public class TestFsck { cluster.shutdown(); // restart the cluster; bring up namenode but not the data nodes - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).format(false).build(); + cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(0).format(false).build(); outStr = runFsck(conf, 1, true, "/"); // expect the result is corrupt assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java index 4e4914b3ab6..5c11c1badf2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; +import org.apache.hadoop.test.PathUtils; import org.junit.Before; import org.junit.Test; @@ -60,7 +61,7 @@ public class TestNameEditsConfigs { short replication = 3; private File base_dir = new File( - System.getProperty("test.build.data", "build/test/data"), "dfs/"); + PathUtils.getTestDir(TestNameEditsConfigs.class), "dfs"); @Before public void setUp() throws IOException { @@ -68,7 +69,7 @@ public class TestNameEditsConfigs { throw new IOException("Cannot remove directory " + base_dir); } } - + void checkImageAndEditsFilesExistence(File dir, boolean shouldHaveImages, boolean shouldHaveEdits) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java index 42f3f5764f9..30d3e71ad8d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.DeleteOp; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.OpInstanceCache; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.test.PathUtils; import org.apache.hadoop.util.StringUtils; import org.junit.Test; @@ -57,8 +58,7 @@ import com.google.common.collect.Sets; public class TestNameNodeRecovery { private static final Log LOG = LogFactory.getLog(TestNameNodeRecovery.class); private static StartupOption recoverStartOpt = StartupOption.RECOVER; - private static final File TEST_DIR = new File( - System.getProperty("test.build.data","build/test/data")); + private static final File TEST_DIR = PathUtils.getTestDir(TestNameNodeRecovery.class); static { recoverStartOpt.setForce(MetaRecoveryContext.FORCE_ALL); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeResourceChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeResourceChecker.java index e73d71aff70..2012b6aabe1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeResourceChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeResourceChecker.java @@ -33,12 +33,14 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem.NameNodeResourceMonitor; import org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.CheckedVolume; +import org.apache.hadoop.test.PathUtils; import org.apache.hadoop.util.Time; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; public class TestNameNodeResourceChecker { + private final static File BASE_DIR = PathUtils.getTestDir(TestNameNodeResourceChecker.class); private Configuration conf; private File baseDir; private File nameDir; @@ -46,8 +48,7 @@ public class TestNameNodeResourceChecker { @Before public void setUp () throws IOException { conf = new Configuration(); - baseDir = new File(System.getProperty("test.build.data")); - nameDir = new File(baseDir, "resource-check-name-dir"); + nameDir = new File(BASE_DIR, "resource-check-name-dir"); nameDir.mkdirs(); conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, nameDir.getAbsolutePath()); } @@ -141,8 +142,8 @@ public class TestNameNodeResourceChecker { @Test public void testChecking2NameDirsOnOneVolume() throws IOException { Configuration conf = new Configuration(); - File nameDir1 = new File(System.getProperty("test.build.data"), "name-dir1"); - File nameDir2 = new File(System.getProperty("test.build.data"), "name-dir2"); + File nameDir1 = new File(BASE_DIR, "name-dir1"); + File nameDir2 = new File(BASE_DIR, "name-dir2"); nameDir1.mkdirs(); nameDir2.mkdirs(); conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, @@ -162,7 +163,7 @@ public class TestNameNodeResourceChecker { @Test public void testCheckingExtraVolumes() throws IOException { Configuration conf = new Configuration(); - File nameDir = new File(System.getProperty("test.build.data"), "name-dir"); + File nameDir = new File(BASE_DIR, "name-dir"); nameDir.mkdirs(); conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, nameDir.getAbsolutePath()); conf.set(DFSConfigKeys.DFS_NAMENODE_CHECKED_VOLUMES_KEY, nameDir.getAbsolutePath()); @@ -182,8 +183,8 @@ public class TestNameNodeResourceChecker { @Test public void testLowResourceVolumePolicy() throws IOException, URISyntaxException { Configuration conf = new Configuration(); - File nameDir1 = new File(System.getProperty("test.build.data"), "name-dir1"); - File nameDir2 = new File(System.getProperty("test.build.data"), "name-dir2"); + File nameDir1 = new File(BASE_DIR, "name-dir1"); + File nameDir2 = new File(BASE_DIR, "name-dir2"); nameDir1.mkdirs(); nameDir2.mkdirs(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java index 6d6dbdf79ce..86323ff5adb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java @@ -57,6 +57,7 @@ import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.hdfs.util.MD5FileUtils; import org.apache.hadoop.io.MD5Hash; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.test.PathUtils; import org.apache.hadoop.util.StringUtils; import org.apache.log4j.Logger; import org.junit.After; @@ -404,8 +405,7 @@ public class TestStartup { Configuration conf = new Configuration(); FileSystem.setDefaultUri(conf, "hdfs://localhost:0"); conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "127.0.0.1:0"); - File base_dir = new File(System.getProperty( - "test.build.data", "build/test/data"), "dfs/"); + File base_dir = new File(PathUtils.getTestDir(getClass()), "dfs/"); conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, new File(base_dir, "name").getPath()); conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, false); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java index 905e3680e0b..1a612e83ec7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java @@ -391,7 +391,8 @@ public class TestStorageRestore { (new File(path3, "current").getAbsolutePath()) : path3.toString(); try { - cluster = new MiniDFSCluster.Builder(config).numDataNodes(0) + cluster = new MiniDFSCluster.Builder(config) + .numDataNodes(0) .manageNameDfsDirs(false).build(); cluster.waitActive(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTransferFsImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTransferFsImage.java index cba634fee92..5a178d19440 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTransferFsImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTransferFsImage.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.http.HttpServer; import org.apache.hadoop.http.HttpServerFunctionalTest; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.test.PathUtils; import org.apache.hadoop.util.StringUtils; import org.junit.Test; import org.mockito.Mockito; @@ -48,8 +49,7 @@ import com.google.common.collect.ImmutableList; public class TestTransferFsImage { - private static final File TEST_DIR = new File( - System.getProperty("test.build.data","build/test/data")); + private static final File TEST_DIR = PathUtils.getTestDir(TestTransferFsImage.class); /** * Regression test for HDFS-1997. Test that, if an exception diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java index 678e03866d5..502c9de4096 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java @@ -56,8 +56,8 @@ public class TestBootstrapStandby { MiniDFSNNTopology topology = new MiniDFSNNTopology() .addNameservice(new MiniDFSNNTopology.NSConf("ns1") - .addNN(new MiniDFSNNTopology.NNConf("nn1").setHttpPort(10001)) - .addNN(new MiniDFSNNTopology.NNConf("nn2").setHttpPort(10002))); + .addNN(new MiniDFSNNTopology.NNConf("nn1").setHttpPort(20001)) + .addNN(new MiniDFSNNTopology.NNConf("nn2").setHttpPort(20002))); cluster = new MiniDFSCluster.Builder(conf) .nnTopology(topology) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSZKFailoverController.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSZKFailoverController.java index ce005b10a14..18972655765 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSZKFailoverController.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSZKFailoverController.java @@ -70,13 +70,13 @@ public class TestDFSZKFailoverController extends ClientBaseWithFixes { CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0); - conf.setInt(DFSConfigKeys.DFS_HA_ZKFC_PORT_KEY + ".ns1.nn1", 10003); - conf.setInt(DFSConfigKeys.DFS_HA_ZKFC_PORT_KEY + ".ns1.nn2", 10004); + conf.setInt(DFSConfigKeys.DFS_HA_ZKFC_PORT_KEY + ".ns1.nn1", 10023); + conf.setInt(DFSConfigKeys.DFS_HA_ZKFC_PORT_KEY + ".ns1.nn2", 10024); MiniDFSNNTopology topology = new MiniDFSNNTopology() .addNameservice(new MiniDFSNNTopology.NSConf("ns1") - .addNN(new MiniDFSNNTopology.NNConf("nn1").setIpcPort(10001)) - .addNN(new MiniDFSNNTopology.NNConf("nn2").setIpcPort(10002))); + .addNN(new MiniDFSNNTopology.NNConf("nn1").setIpcPort(10021)) + .addNN(new MiniDFSNNTopology.NNConf("nn2").setIpcPort(10022))); cluster = new MiniDFSCluster.Builder(conf) .nnTopology(topology) .numDataNodes(0) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java index 8675fa3fc6f..8c61c9237e6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java @@ -124,8 +124,8 @@ public class TestEditLogTailer { // Have to specify IPC ports so the NNs can talk to each other. MiniDFSNNTopology topology = new MiniDFSNNTopology() .addNameservice(new MiniDFSNNTopology.NSConf("ns1") - .addNN(new MiniDFSNNTopology.NNConf("nn1").setIpcPort(10001)) - .addNN(new MiniDFSNNTopology.NNConf("nn2").setIpcPort(10002))); + .addNN(new MiniDFSNNTopology.NNConf("nn1").setIpcPort(10031)) + .addNN(new MiniDFSNNTopology.NNConf("nn2").setIpcPort(10032))); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .nnTopology(topology) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java index 37c0b16fabd..5ec7f7e1c1e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java @@ -76,8 +76,8 @@ public class TestFailureToReadEdits { MiniDFSNNTopology topology = new MiniDFSNNTopology() .addNameservice(new MiniDFSNNTopology.NSConf("ns1") - .addNN(new MiniDFSNNTopology.NNConf("nn1").setHttpPort(10001)) - .addNN(new MiniDFSNNTopology.NNConf("nn2").setHttpPort(10002))); + .addNN(new MiniDFSNNTopology.NNConf("nn1").setHttpPort(10041)) + .addNN(new MiniDFSNNTopology.NNConf("nn2").setHttpPort(10042))); cluster = new MiniDFSCluster.Builder(conf) .nnTopology(topology) .numDataNodes(0) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAFsck.java index 49d89592b8a..4f848dcf834 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAFsck.java @@ -52,8 +52,8 @@ public class TestHAFsck { // need some HTTP ports MiniDFSNNTopology topology = new MiniDFSNNTopology() .addNameservice(new MiniDFSNNTopology.NSConf("ha-nn-uri-0") - .addNN(new MiniDFSNNTopology.NNConf("nn1").setHttpPort(10001)) - .addNN(new MiniDFSNNTopology.NNConf("nn2").setHttpPort(10002))); + .addNN(new MiniDFSNNTopology.NNConf("nn1").setHttpPort(10051)) + .addNN(new MiniDFSNNTopology.NNConf("nn2").setHttpPort(10052))); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .nnTopology(topology) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java index dff28740690..3ff5d54dc66 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java @@ -89,8 +89,8 @@ public class TestStandbyCheckpoints { MiniDFSNNTopology topology = new MiniDFSNNTopology() .addNameservice(new MiniDFSNNTopology.NSConf("ns1") - .addNN(new MiniDFSNNTopology.NNConf("nn1").setHttpPort(10001)) - .addNN(new MiniDFSNNTopology.NNConf("nn2").setHttpPort(10002))); + .addNN(new MiniDFSNNTopology.NNConf("nn1").setHttpPort(10061)) + .addNN(new MiniDFSNNTopology.NNConf("nn2").setHttpPort(10062))); cluster = new MiniDFSCluster.Builder(conf) .nnTopology(topology) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java index 1d189a108de..e6c9a3f3967 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes; import org.apache.hadoop.hdfs.server.namenode.OfflineEditsViewerHelper; import org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer.Flags; +import org.apache.hadoop.test.PathUtils; import org.junit.Before; import org.junit.Test; @@ -53,7 +54,7 @@ public class TestOfflineEditsViewer { } private static String buildDir = - System.getProperty("test.build.data", "build/test/data"); + PathUtils.getTestDirName(TestOfflineEditsViewer.class); private static String cacheDir = System.getProperty("test.cache.data", "build/test/cache"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestDelimitedImageVisitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestDelimitedImageVisitor.java index a5501d97547..c7d3b31dacf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestDelimitedImageVisitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestDelimitedImageVisitor.java @@ -27,6 +27,7 @@ import java.io.FileReader; import java.io.IOException; import org.apache.hadoop.hdfs.tools.offlineImageViewer.ImageVisitor.ImageElement; +import org.apache.hadoop.test.PathUtils; import org.junit.Test; /** @@ -34,7 +35,7 @@ import org.junit.Test; * on predetermined inputs */ public class TestDelimitedImageVisitor { - private static String ROOT = System.getProperty("test.build.data","/tmp"); + private static String ROOT = PathUtils.getTestDirName(TestDelimitedImageVisitor.class); private static final String delim = "--"; // Record an element in the visitor and build the expected line in the output diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java index 50e816417d3..11aa3b821f0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java @@ -53,6 +53,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.test.PathUtils; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -88,8 +89,7 @@ public class TestOfflineImageViewer { final static HashMap writtenFiles = new HashMap(); - private static String ROOT = System.getProperty("test.build.data", - "build/test/data"); + private static String ROOT = PathUtils.getTestDirName(TestOfflineImageViewer.class); // Create a populated namespace for later testing. Save its contents to a // data structure and store its fsimage location. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestAtomicFileOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestAtomicFileOutputStream.java index ebbb4e22701..2a9465aa90c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestAtomicFileOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestAtomicFileOutputStream.java @@ -30,6 +30,7 @@ import java.io.OutputStream; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.test.PathUtils; import org.junit.Before; import org.junit.Test; @@ -40,10 +41,7 @@ public class TestAtomicFileOutputStream { private static final String TEST_STRING = "hello world"; private static final String TEST_STRING_2 = "goodbye world"; - private static File BASE_DIR = new File( - System.getProperty("test.build.data", "build/test/data")); - private static File TEST_DIR = new File(BASE_DIR, - TestAtomicFileOutputStream.class.getName()); + private static File TEST_DIR = PathUtils.getTestDir(TestAtomicFileOutputStream.class); private static File DST_FILE = new File(TEST_DIR, "test.txt"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestMD5FileUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestMD5FileUtils.java index 6f5b1613360..35fa46d20cd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestMD5FileUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestMD5FileUtils.java @@ -29,14 +29,12 @@ import java.io.IOException; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.io.MD5Hash; +import org.apache.hadoop.test.PathUtils; import org.junit.Before; import org.junit.Test; public class TestMD5FileUtils { - private static final File TEST_DIR_ROOT = new File( - System.getProperty("test.build.data","build/test/data")); - private static final File TEST_DIR = new File(TEST_DIR_ROOT, - "TestMD5FileUtils"); + private static final File TEST_DIR = PathUtils.getTestDir(TestMD5FileUtils.class); private static final File TEST_FILE = new File(TEST_DIR, "testMd5File.dat"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTimeouts.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTimeouts.java index 2071f6feb87..7a007a05928 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTimeouts.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTimeouts.java @@ -58,7 +58,6 @@ public class TestWebHdfsTimeouts { private static final int CLIENTS_TO_CONSUME_BACKLOG = 100; private static final int CONNECTION_BACKLOG = 1; - private static final int INITIAL_SOCKET_TIMEOUT = URLUtils.SOCKET_TIMEOUT; private static final int SHORT_SOCKET_TIMEOUT = 5; private static final int TEST_TIMEOUT = 10000; @@ -67,20 +66,22 @@ public class TestWebHdfsTimeouts { private InetSocketAddress nnHttpAddress; private ServerSocket serverSocket; private Thread serverThread; + private URLConnectionFactory connectionFactory = new URLConnectionFactory(SHORT_SOCKET_TIMEOUT); @Before public void setUp() throws Exception { - URLUtils.SOCKET_TIMEOUT = SHORT_SOCKET_TIMEOUT; Configuration conf = WebHdfsTestUtil.createConf(); nnHttpAddress = NameNode.getHttpAddress(conf); serverSocket = new ServerSocket(nnHttpAddress.getPort(), CONNECTION_BACKLOG); fs = WebHdfsTestUtil.getWebHdfsFileSystem(conf); + fs.connectionFactory = connectionFactory; clients = new ArrayList(); serverThread = null; } @After public void tearDown() throws Exception { + fs.connectionFactory = URLConnectionFactory.DEFAULT_CONNECTION_FACTORY; IOUtils.cleanup(LOG, clients.toArray(new SocketChannel[clients.size()])); IOUtils.cleanup(LOG, fs); if (serverSocket != null) { @@ -240,7 +241,7 @@ public class TestWebHdfsTimeouts { */ private void startSingleTemporaryRedirectResponseThread( final boolean consumeConnectionBacklog) { - URLUtils.SOCKET_TIMEOUT = INITIAL_SOCKET_TIMEOUT; + fs.connectionFactory = URLConnectionFactory.DEFAULT_CONNECTION_FACTORY; serverThread = new Thread() { @Override public void run() { @@ -254,7 +255,7 @@ public class TestWebHdfsTimeouts { clientSocket = serverSocket.accept(); // Immediately setup conditions for subsequent connections. - URLUtils.SOCKET_TIMEOUT = SHORT_SOCKET_TIMEOUT; + fs.connectionFactory = connectionFactory; if (consumeConnectionBacklog) { consumeConnectionBacklog(); } diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 8dee23a467e..bc7e802a2db 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -713,7 +713,7 @@ org.apache.maven.plugins maven-surefire-plugin - 2.12.3 + 2.16 org.apache.maven.plugins @@ -827,7 +827,7 @@ org.apache.maven.plugins maven-surefire-plugin - always + false 900 -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError @@ -1002,23 +1002,5 @@ - From e10dbf41bc183aa8629e067d381e1c8ba9dae95a Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Fri, 6 Sep 2013 03:14:20 +0000 Subject: [PATCH 02/21] HDFS-4491. Add/delete files missed in prior commit. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1520482 13f79535-47bb-0310-9956-ffa450edef68 --- ...RLUtils.java => URLConnectionFactory.java} | 20 ++++--- .../org/apache/hadoop/test/PathUtils.java | 54 +++++++++++++++++++ 2 files changed, 68 insertions(+), 6 deletions(-) rename hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/{URLUtils.java => URLConnectionFactory.java} (73%) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/test/PathUtils.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/URLUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java similarity index 73% rename from hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/URLUtils.java rename to hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java index 09feaf5bec4..54aab04e58c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/URLUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java @@ -30,19 +30,27 @@ import org.apache.hadoop.classification.InterfaceStability; */ @InterfaceAudience.LimitedPrivate({"HDFS"}) @InterfaceStability.Unstable -public class URLUtils { +public class URLConnectionFactory { /** * Timeout for socket connects and reads */ - public static int SOCKET_TIMEOUT = 1*60*1000; // 1 minute + public final static int DEFAULT_SOCKET_TIMEOUT = 1*60*1000; // 1 minute + public static final URLConnectionFactory DEFAULT_CONNECTION_FACTORY = new URLConnectionFactory(DEFAULT_SOCKET_TIMEOUT); + + private int socketTimeout; + + public URLConnectionFactory(int socketTimeout) { + this.socketTimeout = socketTimeout; + } + /** * Opens a url with read and connect timeouts * @param url to open * @return URLConnection * @throws IOException */ - public static URLConnection openConnection(URL url) throws IOException { + public URLConnection openConnection(URL url) throws IOException { URLConnection connection = url.openConnection(); setTimeouts(connection); return connection; @@ -53,8 +61,8 @@ public class URLUtils { * * @param connection URLConnection to set */ - static void setTimeouts(URLConnection connection) { - connection.setConnectTimeout(SOCKET_TIMEOUT); - connection.setReadTimeout(SOCKET_TIMEOUT); + public void setTimeouts(URLConnection connection) { + connection.setConnectTimeout(socketTimeout); + connection.setReadTimeout(socketTimeout); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/test/PathUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/test/PathUtils.java new file mode 100644 index 00000000000..2ee4aa1390b --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/test/PathUtils.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.test; + +import java.io.File; + +import org.apache.hadoop.fs.Path; + +public class PathUtils { + + public static Path getTestPath(Class caller) { + return getTestPath(caller, true); + } + + public static Path getTestPath(Class caller, boolean create) { + return new Path(getTestDirName(caller)); + } + + public static File getTestDir(Class caller) { + return getTestDir(caller, true); + } + + public static File getTestDir(Class caller, boolean create) { + File dir = new File(System.getProperty("test.build.data", "/tmp"), caller.getSimpleName()); + if (create) { + dir.mkdirs(); + } + return dir; + } + + public static String getTestDirName(Class caller) { + return getTestDirName(caller, true); + } + + public static String getTestDirName(Class caller, boolean create) { + return getTestDir(caller, create).getAbsolutePath(); + } + +} From a62839195548a632b1b6197456b9adf95127427c Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Fri, 6 Sep 2013 05:23:54 +0000 Subject: [PATCH 03/21] HDFS-5164. DeleteSnapshot should check if OperationCategory.WRITE is possible before taking write lock (contributed by Colin Patrick McCabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1520492 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 1 + 2 files changed, 4 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 2015f2954f4..dd445c2fd97 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -283,6 +283,9 @@ Release 2.3.0 - UNRELEASED HDFS-4816. transitionToActive blocks if the SBN is doing checkpoint image transfer. (Andrew Wang) + HDFS-5164. deleteSnapshot should check if OperationCategory.WRITE is + possible before taking write lock. (Colin Patrick McCabe) + Release 2.1.1-beta - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 0ed66324e45..42965ed189e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -6767,6 +6767,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, return; // Return previous response } boolean success = false; + checkOperation(OperationCategory.WRITE); writeLock(); try { checkOperation(OperationCategory.WRITE); From 36d0b822ef46fcacdb773abbdd3e81386eb4d63c Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Fri, 6 Sep 2013 17:17:30 +0000 Subject: [PATCH 04/21] HDS-5118. Provide testing support for DFSClient to drop RPC responses. Contributed by Jing Zhao. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1520637 13f79535-47bb-0310-9956-ffa450edef68 --- .../io/retry/LossyRetryInvocationHandler.java | 62 ++++++++++++++++++ .../io/retry/RetryInvocationHandler.java | 2 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../org/apache/hadoop/hdfs/DFSClient.java | 33 ++++++++-- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 6 ++ .../apache/hadoop/hdfs/NameNodeProxies.java | 64 +++++++++++++++++++ 6 files changed, 162 insertions(+), 8 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/LossyRetryInvocationHandler.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/LossyRetryInvocationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/LossyRetryInvocationHandler.java new file mode 100644 index 00000000000..df5895553a6 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/LossyRetryInvocationHandler.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.retry; + +import java.lang.reflect.Method; +import java.net.UnknownHostException; + +import org.apache.hadoop.classification.InterfaceAudience; + +/** + * A dummy invocation handler extending RetryInvocationHandler. It drops the + * first N number of responses. This invocation handler is only used for testing. + */ +@InterfaceAudience.Private +public class LossyRetryInvocationHandler extends RetryInvocationHandler { + private final int numToDrop; + private static final ThreadLocal RetryCount = + new ThreadLocal(); + + public LossyRetryInvocationHandler(int numToDrop, + FailoverProxyProvider proxyProvider, RetryPolicy retryPolicy) { + super(proxyProvider, retryPolicy); + this.numToDrop = numToDrop; + } + + @Override + public Object invoke(Object proxy, Method method, Object[] args) + throws Throwable { + RetryCount.set(0); + return super.invoke(proxy, method, args); + } + + @Override + protected Object invokeMethod(Method method, Object[] args) throws Throwable { + Object result = super.invokeMethod(method, args); + int retryCount = RetryCount.get(); + if (retryCount < this.numToDrop) { + RetryCount.set(++retryCount); + LOG.info("Drop the response. Current retryCount == " + retryCount); + throw new UnknownHostException("Fake Exception"); + } else { + LOG.info("retryCount == " + retryCount + + ". It's time to normally process the response"); + return result; + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java index 974bac91eb4..51dd46a8f9e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java @@ -63,7 +63,7 @@ public class RetryInvocationHandler implements RpcInvocationHandler { this(proxyProvider, retryPolicy, Collections.emptyMap()); } - RetryInvocationHandler(FailoverProxyProvider proxyProvider, + protected RetryInvocationHandler(FailoverProxyProvider proxyProvider, RetryPolicy defaultPolicy, Map methodNameToPolicyMap) { this.proxyProvider = proxyProvider; diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index dd445c2fd97..dd86e0e4e4a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -244,6 +244,9 @@ Release 2.3.0 - UNRELEASED NEW FEATURES + HDFS-5118. Provide testing support for DFSClient to drop RPC responses. + (jing9) + IMPROVEMENTS HDFS-4657. Limit the number of blocks logged by the NN after a block diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 511df17b472..d1cc7848916 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -27,6 +27,9 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_BLOCK_WRITE_RETRIE import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_BLOCK_WRITE_RETRIES_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHED_CONN_RETRY_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHED_CONN_RETRY_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHE_DROP_BEHIND_READS; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHE_DROP_BEHIND_WRITES; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHE_READAHEAD; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT; @@ -44,9 +47,6 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_EXPIR import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHE_DROP_BEHIND_READS; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHE_DROP_BEHIND_WRITES; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHE_READAHEAD; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT; @@ -100,6 +100,7 @@ import org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum; import org.apache.hadoop.fs.Options; import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.fs.ParentNotDirectoryException; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.fs.VolumeId; import org.apache.hadoop.fs.permission.FsPermission; @@ -113,13 +114,13 @@ import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata; import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException; +import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; import org.apache.hadoop.hdfs.protocol.UnresolvedPathException; @@ -144,6 +145,7 @@ import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.MD5Hash; import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.retry.LossyRetryInvocationHandler; import org.apache.hadoop.ipc.Client; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RemoteException; @@ -451,7 +453,11 @@ public class DFSClient implements java.io.Closeable { /** * Create a new DFSClient connected to the given nameNodeUri or rpcNamenode. - * Exactly one of nameNodeUri or rpcNamenode must be null. + * If HA is enabled and a positive value is set for + * {@link DFSConfigKeys#DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY} in the + * configuration, the DFSClient will use {@link LossyRetryInvocationHandler} + * as its RetryInvocationHandler. Otherwise one of nameNodeUri or rpcNamenode + * must be null. */ @VisibleForTesting public DFSClient(URI nameNodeUri, ClientProtocol rpcNamenode, @@ -475,7 +481,20 @@ public class DFSClient implements java.io.Closeable { this.clientName = "DFSClient_" + dfsClientConf.taskId + "_" + DFSUtil.getRandom().nextInt() + "_" + Thread.currentThread().getId(); - if (rpcNamenode != null) { + int numResponseToDrop = conf.getInt( + DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY, + DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_DEFAULT); + if (numResponseToDrop > 0) { + // This case is used for testing. + LOG.warn(DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY + + " is set to " + numResponseToDrop + + ", this hacked client will proactively drop responses"); + NameNodeProxies.ProxyAndInfo proxyInfo = NameNodeProxies + .createProxyWithLossyRetryHandler(conf, nameNodeUri, + ClientProtocol.class, numResponseToDrop); + this.dtService = proxyInfo.getDelegationTokenService(); + this.namenode = proxyInfo.getProxy(); + } else if (rpcNamenode != null) { // This case is used for testing. Preconditions.checkArgument(nameNodeUri == null); this.namenode = rpcNamenode; @@ -514,7 +533,7 @@ public class DFSClient implements java.io.Closeable { this.defaultWriteCachingStrategy = new CachingStrategy(writeDropBehind, readahead); } - + /** * Return the socket addresses to use with each configured * local interface. Local interfaces may be specified by IP diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index a66ec939613..b4d67ca19d2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -497,6 +497,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final long DFS_NAMENODE_RETRY_CACHE_EXPIRYTIME_MILLIS_DEFAULT = 600000; // 10 minutes public static final String DFS_NAMENODE_RETRY_CACHE_HEAP_PERCENT_KEY = "dfs.namenode.retrycache.heap.percent"; public static final float DFS_NAMENODE_RETRY_CACHE_HEAP_PERCENT_DEFAULT = 0.03f; + + // The number of NN response dropped by client proactively in each RPC call. + // For testing NN retry cache, we can set this property with positive value. + public static final String DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY = "dfs.client.test.drop.namenode.response.number"; + public static final int DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_DEFAULT = 0; + // Hidden configuration undocumented in hdfs-site. xml // Timeout to wait for block receiver and responder thread to stop diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java index eb745b8bb7d..41dac6a80f6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java @@ -17,10 +17,18 @@ */ package org.apache.hadoop.hdfs; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY; import java.io.IOException; import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Proxy; import java.net.InetSocketAddress; import java.net.URI; import java.util.HashMap; @@ -48,6 +56,7 @@ import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.retry.DefaultFailoverProxyProvider; import org.apache.hadoop.io.retry.FailoverProxyProvider; +import org.apache.hadoop.io.retry.LossyRetryInvocationHandler; import org.apache.hadoop.io.retry.RetryPolicies; import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.io.retry.RetryProxy; @@ -144,6 +153,61 @@ public class NameNodeProxies { return new ProxyAndInfo(proxy, dtService); } } + + /** + * Generate a dummy namenode proxy instance that utilizes our hacked + * {@link LossyRetryInvocationHandler}. Proxy instance generated using this + * method will proactively drop RPC responses. Currently this method only + * support HA setup. IllegalStateException will be thrown if the given + * configuration is not for HA. + * + * @param config the configuration containing the required IPC + * properties, client failover configurations, etc. + * @param nameNodeUri the URI pointing either to a specific NameNode + * or to a logical nameservice. + * @param xface the IPC interface which should be created + * @param numResponseToDrop The number of responses to drop for each RPC call + * @return an object containing both the proxy and the associated + * delegation token service it corresponds to + * @throws IOException if there is an error creating the proxy + */ + @SuppressWarnings("unchecked") + public static ProxyAndInfo createProxyWithLossyRetryHandler( + Configuration config, URI nameNodeUri, Class xface, + int numResponseToDrop) throws IOException { + Preconditions.checkArgument(numResponseToDrop > 0); + Class> failoverProxyProviderClass = + getFailoverProxyProviderClass(config, nameNodeUri, xface); + if (failoverProxyProviderClass != null) { // HA case + FailoverProxyProvider failoverProxyProvider = + createFailoverProxyProvider(config, failoverProxyProviderClass, + xface, nameNodeUri); + int delay = config.getInt( + DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY, + DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT); + int maxCap = config.getInt( + DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY, + DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT); + int maxFailoverAttempts = config.getInt( + DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY, + DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT); + InvocationHandler dummyHandler = new LossyRetryInvocationHandler( + numResponseToDrop, failoverProxyProvider, + RetryPolicies.failoverOnNetworkException( + RetryPolicies.TRY_ONCE_THEN_FAIL, + Math.max(numResponseToDrop + 1, maxFailoverAttempts), delay, + maxCap)); + + T proxy = (T) Proxy.newProxyInstance( + failoverProxyProvider.getInterface().getClassLoader(), + new Class[] { xface }, dummyHandler); + Text dtService = HAUtil.buildTokenServiceForLogicalUri(nameNodeUri); + return new ProxyAndInfo(proxy, dtService); + } else { + throw new IllegalStateException("Currently creating proxy using " + + "LossyRetryInvocationHandler requires NN HA setup"); + } + } /** * Creates an explicitly non-HA-enabled proxy object. Most of the time you From 4065c842ab72abfd6efcf67b195cc51cfbf532e7 Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Fri, 6 Sep 2013 18:16:30 +0000 Subject: [PATCH 05/21] Move HDFS-5118 to 2.1.1-beta section. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1520650 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index dd86e0e4e4a..f5b2cc17093 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -244,9 +244,6 @@ Release 2.3.0 - UNRELEASED NEW FEATURES - HDFS-5118. Provide testing support for DFSClient to drop RPC responses. - (jing9) - IMPROVEMENTS HDFS-4657. Limit the number of blocks logged by the NN after a block @@ -324,6 +321,9 @@ Release 2.1.1-beta - UNRELEASED HDFS-5136 MNT EXPORT should give the full group list which can mount the exports (brandonli) + HDFS-5118. Provide testing support for DFSClient to drop RPC responses. + (jing9) + IMPROVEMENTS HDFS-4513. Clarify in the WebHDFS REST API that all JSON respsonses may From fc9599811712b4e5e32304afa2f22c9ec6a2e21b Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Fri, 6 Sep 2013 18:30:00 +0000 Subject: [PATCH 06/21] Fixing CHANGES.txt for YARN-758 as it is now merged into branch-2.1-beta. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1520659 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index f3cd1c30726..7fe2e0af20c 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -32,9 +32,6 @@ Release 2.3.0 - UNRELEASED BUG FIXES - YARN-758. Augment MockNM to use multiple cores (Karthik Kambatla via - Sandy Ryza) - YARN-1060. Two tests in TestFairScheduler are missing @Test annotation (Niranjan Singh via Sandy Ryza) @@ -89,6 +86,9 @@ Release 2.1.1-beta - UNRELEASED YARN-1065. NM should provide AuxillaryService data to the container (Xuan Gong via bikas) + YARN-758. Augment MockNM to use multiple cores (Karthik Kambatla via + Sandy Ryza) + OPTIMIZATIONS BUG FIXES From 6431192c0ee00ecfe578b270889b0c7a0a9cb8c8 Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Fri, 6 Sep 2013 19:05:26 +0000 Subject: [PATCH 07/21] HDFS-4879. Add BlockedArrayList collection to avoid CMS full GCs (Contributed by Todd Lipcon) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1520667 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hdfs/server/namenode/FSDirectory.java | 5 +- .../hdfs/server/namenode/FSEditLogLoader.java | 4 +- .../hdfs/server/namenode/FSNamesystem.java | 17 +- .../hadoop/hdfs/server/namenode/INode.java | 3 +- .../hadoop/hdfs/util/ChunkedArrayList.java | 171 ++++++++++++++++++ .../hdfs/util/TestChunkedArrayList.java | 93 ++++++++++ 7 files changed, 281 insertions(+), 15 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/ChunkedArrayList.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestChunkedArrayList.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index f5b2cc17093..3dbcde5d619 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -269,6 +269,9 @@ Release 2.3.0 - UNRELEASED HDFS-4491. Parallel testing HDFS. (Andrey Klochkov via cnauroth) + HDFS-4879. Add "blocked ArrayList" collection to avoid CMS full GCs + (Todd Lipcon via Colin Patrick McCabe) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index 51642a8b23a..4af1f963fa3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -69,6 +69,7 @@ import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectoryWithSnapsho import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot.Root; import org.apache.hadoop.hdfs.util.ByteArray; +import org.apache.hadoop.hdfs.util.ChunkedArrayList; import org.apache.hadoop.hdfs.util.ReadOnlyList; import com.google.common.annotations.VisibleForTesting; @@ -949,7 +950,7 @@ public class FSDirectory implements Closeable { if (removedDst != null) { undoRemoveDst = false; BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo(); - List removedINodes = new ArrayList(); + List removedINodes = new ChunkedArrayList(); filesDeleted = removedDst.cleanSubtree(null, dstIIP.getLatestSnapshot(), collectedBlocks, removedINodes, true) .get(Quota.NAMESPACE); @@ -1363,7 +1364,7 @@ public class FSDirectory implements Closeable { QuotaExceededException, SnapshotAccessControlException { assert hasWriteLock(); BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo(); - List removedINodes = new ArrayList(); + List removedINodes = new ChunkedArrayList(); final INodesInPath inodesInPath = rootDir.getINodesInPath4Write( normalizePath(src), false); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java index f19b15cd9f6..03a1dbc1fbd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java @@ -22,7 +22,6 @@ import static org.apache.hadoop.util.Time.now; import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; -import java.util.ArrayList; import java.util.Arrays; import java.util.EnumMap; import java.util.List; @@ -75,6 +74,7 @@ import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase; import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress; import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress.Counter; import org.apache.hadoop.hdfs.server.namenode.startupprogress.Step; +import org.apache.hadoop.hdfs.util.ChunkedArrayList; import org.apache.hadoop.hdfs.util.Holder; import com.google.common.base.Joiner; @@ -582,7 +582,7 @@ public class FSEditLogLoader { case OP_DELETE_SNAPSHOT: { DeleteSnapshotOp deleteSnapshotOp = (DeleteSnapshotOp) op; BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo(); - List removedINodes = new ArrayList(); + List removedINodes = new ChunkedArrayList(); fsNamesys.getSnapshotManager().deleteSnapshot( deleteSnapshotOp.snapshotRoot, deleteSnapshotOp.snapshotName, collectedBlocks, removedINodes); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 42965ed189e..884a14b8959 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -201,6 +201,7 @@ import org.apache.hadoop.hdfs.server.protocol.NamenodeCommand; import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo; +import org.apache.hadoop.hdfs.util.ChunkedArrayList; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.RetryCache; @@ -3130,7 +3131,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, throws AccessControlException, SafeModeException, UnresolvedLinkException, IOException { BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo(); - List removedINodes = new ArrayList(); + List removedINodes = new ChunkedArrayList(); FSPermissionChecker pc = getPermissionChecker(); checkOperation(OperationCategory.WRITE); byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src); @@ -3184,21 +3185,17 @@ public class FSNamesystem implements Namesystem, FSClusterStats, * of blocks that need to be removed from blocksMap */ void removeBlocks(BlocksMapUpdateInfo blocks) { - int start = 0; - int end = 0; List toDeleteList = blocks.getToDeleteList(); - while (start < toDeleteList.size()) { - end = BLOCK_DELETION_INCREMENT + start; - end = end > toDeleteList.size() ? toDeleteList.size() : end; + Iterator iter = toDeleteList.iterator(); + while (iter.hasNext()) { writeLock(); try { - for (int i = start; i < end; i++) { - blockManager.removeBlock(toDeleteList.get(i)); + for (int i = 0; i < BLOCK_DELETION_INCREMENT && iter.hasNext(); i++) { + blockManager.removeBlock(iter.next()); } } finally { writeUnlock(); } - start = end; } } @@ -6778,7 +6775,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, checkOwner(pc, snapshotRoot); BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo(); - List removedINodes = new ArrayList(); + List removedINodes = new ChunkedArrayList(); dir.writeLock(); try { snapshotManager.deleteSnapshot(snapshotRoot, snapshotName, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java index 94ad7a8479c..977c801013e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithName; import org.apache.hadoop.hdfs.server.namenode.snapshot.FileWithSnapshot; import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectoryWithSnapshot; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; +import org.apache.hadoop.hdfs.util.ChunkedArrayList; import org.apache.hadoop.hdfs.util.Diff; import org.apache.hadoop.util.StringUtils; @@ -707,7 +708,7 @@ public abstract class INode implements INodeAttributes, Diff.Element { } public BlocksMapUpdateInfo() { - toDeleteList = new ArrayList(); + toDeleteList = new ChunkedArrayList(); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/ChunkedArrayList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/ChunkedArrayList.java new file mode 100644 index 00000000000..89a0db6eb47 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/ChunkedArrayList.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.util; + +import java.util.AbstractList; +import java.util.Iterator; +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; + +/** + * Simplified List implementation which stores elements as a list + * of chunks, each chunk having a maximum size. This improves over + * using an ArrayList in that creating a large list will never require + * a large amount of contiguous heap space -- thus reducing the likelihood + * of triggering a CMS compaction pause due to heap fragmentation. + * + * The first chunks allocated are small, but each additional chunk is + * 50% larger than the previous, ramping up to a configurable maximum + * chunk size. Reasonable defaults are provided which should be a good + * balance between not making any large allocations while still retaining + * decent performance. + * + * This currently only supports a small subset of List operations -- + * namely addition and iteration. + */ +@InterfaceAudience.Private +public class ChunkedArrayList extends AbstractList { + + /** + * The chunks which make up the full list. + */ + private final List> chunks = Lists.newArrayList(); + + /** + * Cache of the last element in the 'chunks' array above. + * This speeds up the add operation measurably. + */ + private List lastChunk = null; + + /** + * The capacity with which the last chunk was allocated. + */ + private int lastChunkCapacity; + + /** + * The capacity of the first chunk to allocate in a cleared list. + */ + private final int initialChunkCapacity; + + /** + * The maximum number of elements for any chunk. + */ + private final int maxChunkSize; + + /** + * Total number of elements in the list. + */ + private int size; + + /** + * Default initial size is 6 elements, since typical minimum object + * size is 64 bytes, and this leaves enough space for the object + * header. + */ + private static final int DEFAULT_INITIAL_CHUNK_CAPACITY = 6; + + /** + * Default max size is 8K elements - which, at 8 bytes per element + * should be about 64KB -- small enough to easily fit in contiguous + * free heap space even with a fair amount of fragmentation. + */ + private static final int DEFAULT_MAX_CHUNK_SIZE = 8*1024; + + + public ChunkedArrayList() { + this(DEFAULT_INITIAL_CHUNK_CAPACITY, DEFAULT_MAX_CHUNK_SIZE); + } + + /** + * @param initialChunkCapacity the capacity of the first chunk to be + * allocated + * @param maxChunkSize the maximum size of any chunk allocated + */ + public ChunkedArrayList(int initialChunkCapacity, int maxChunkSize) { + Preconditions.checkArgument(maxChunkSize >= initialChunkCapacity); + this.initialChunkCapacity = initialChunkCapacity; + this.maxChunkSize = maxChunkSize; + } + + @Override + public Iterator iterator() { + return Iterables.concat(chunks).iterator(); + } + + @Override + public boolean add(T e) { + if (lastChunk == null) { + addChunk(initialChunkCapacity); + } else if (lastChunk.size() >= lastChunkCapacity) { + int newCapacity = lastChunkCapacity + (lastChunkCapacity >> 1); + addChunk(Math.min(newCapacity, maxChunkSize)); + } + size++; + return lastChunk.add(e); + } + + @Override + public void clear() { + chunks.clear(); + lastChunk = null; + lastChunkCapacity = 0; + size = 0; + } + + private void addChunk(int capacity) { + lastChunk = Lists.newArrayListWithCapacity(capacity); + chunks.add(lastChunk); + lastChunkCapacity = capacity; + } + + @Override + public boolean isEmpty() { + return size == 0; + } + + @Override + public int size() { + return size; + } + + @VisibleForTesting + int getNumChunks() { + return chunks.size(); + } + + @VisibleForTesting + int getMaxChunkSize() { + int size = 0; + for (List chunk : chunks) { + size = Math.max(size, chunk.size()); + } + return size; + } + + @Override + public T get(int arg0) { + throw new UnsupportedOperationException( + this.getClass().getName() + " does not support random access"); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestChunkedArrayList.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestChunkedArrayList.java new file mode 100644 index 00000000000..a1e49ccee28 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestChunkedArrayList.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.util; + +import static org.junit.Assert.*; + +import java.util.ArrayList; + +import org.junit.Test; + +import com.google.common.base.Stopwatch; + +public class TestChunkedArrayList { + + @Test + public void testBasics() { + final int N_ELEMS = 100000; + ChunkedArrayList l = new ChunkedArrayList(); + assertTrue(l.isEmpty()); + // Insert a bunch of elements. + for (int i = 0; i < N_ELEMS; i++) { + l.add(i); + } + assertFalse(l.isEmpty()); + assertEquals(N_ELEMS, l.size()); + + // Check that it got chunked. + assertTrue(l.getNumChunks() > 10); + assertEquals(8192, l.getMaxChunkSize()); + } + + @Test + public void testIterator() { + ChunkedArrayList l = new ChunkedArrayList(); + for (int i = 0; i < 30000; i++) { + l.add(i); + } + + int i = 0; + for (int fromList : l) { + assertEquals(i, fromList); + i++; + } + } + + @Test + public void testPerformance() { + String obj = "hello world"; + + final int numElems = 1000000; + final int numTrials = 5; + + for (int trial = 0; trial < numTrials; trial++) { + System.gc(); + { + ArrayList arrayList = new ArrayList(); + Stopwatch sw = new Stopwatch(); + sw.start(); + for (int i = 0; i < numElems; i++) { + arrayList.add(obj); + } + System.out.println(" ArrayList " + sw.elapsedMillis()); + } + + // test ChunkedArrayList + System.gc(); + { + ChunkedArrayList chunkedList = new ChunkedArrayList(); + Stopwatch sw = new Stopwatch(); + sw.start(); + for (int i = 0; i < numElems; i++) { + chunkedList.add(obj); + } + System.out.println("ChunkedArrayList " + sw.elapsedMillis()); + } + } + } +} From 6cfcd391652ffe03fa95fdf2e29b1d10fcae4ea6 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Eagles Date: Fri, 6 Sep 2013 19:40:02 +0000 Subject: [PATCH 08/21] HADOOP-8704. add request logging to jetty/httpserver (jeagles) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1520674 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 + .../src/main/conf/log4j.properties | 25 +++++ .../apache/hadoop/http/HttpRequestLog.java | 94 +++++++++++++++++++ .../hadoop/http/HttpRequestLogAppender.java | 62 ++++++++++++ .../org/apache/hadoop/http/HttpServer.java | 16 +++- .../hadoop/http/HttpServerFunctionalTest.java | 6 +- .../hadoop/http/TestHttpRequestLog.java | 47 ++++++++++ .../http/TestHttpRequestLogAppender.java | 37 ++++++++ .../hadoop/http/TestHttpServerLifecycle.java | 22 +++++ 9 files changed, 309 insertions(+), 2 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLogAppender.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLogAppender.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index c2beef64124..22ea0bee75c 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -2117,6 +2117,8 @@ Release 0.23.10 - UNRELEASED HADOOP-9686. Easy access to final parameters in Configuration (Jason Lowe via jeagles) + HADOOP-8704. add request logging to jetty/httpserver (jeagles) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties index d436db9df3b..8ef5eb4635b 100644 --- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties +++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties @@ -240,3 +240,28 @@ log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n #log4j.appender.HSAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n #log4j.appender.HSAUDIT.DatePattern=.yyyy-MM-dd +# Http Server Request Logs +#log4j.logger.http.requests.namenode=INFO,namenoderequestlog +#log4j.appender.namenoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender +#log4j.appender.namenoderequestlog.Filename=${hadoop.log.dir}/jetty-namenode-yyyy_mm_dd.log +#log4j.appender.namenoderequestlog.RetainDays=3 + +#log4j.logger.http.requests.datanode=INFO,datanoderequestlog +#log4j.appender.datanoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender +#log4j.appender.datanoderequestlog.Filename=${hadoop.log.dir}/jetty-datanode-yyyy_mm_dd.log +#log4j.appender.datanoderequestlog.RetainDays=3 + +#log4j.logger.http.requests.resourcemanager=INFO,resourcemanagerrequestlog +#log4j.appender.resourcemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender +#log4j.appender.resourcemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-resourcemanager-yyyy_mm_dd.log +#log4j.appender.resourcemanagerrequestlog.RetainDays=3 + +#log4j.logger.http.requests.jobhistory=INFO,jobhistoryrequestlog +#log4j.appender.jobhistoryrequestlog=org.apache.hadoop.http.HttpRequestLogAppender +#log4j.appender.jobhistoryrequestlog.Filename=${hadoop.log.dir}/jetty-jobhistory-yyyy_mm_dd.log +#log4j.appender.jobhistoryrequestlog.RetainDays=3 + +#log4j.logger.http.requests.nodemanager=INFO,nodemanagerrequestlog +#log4j.appender.nodemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender +#log4j.appender.nodemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-nodemanager-yyyy_mm_dd.log +#log4j.appender.nodemanagerrequestlog.RetainDays=3 diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java new file mode 100644 index 00000000000..eb8968bcc49 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.http; + +import java.util.HashMap; + +import org.apache.commons.logging.impl.Log4JLogger; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogConfigurationException; +import org.apache.commons.logging.LogFactory; +import org.apache.log4j.Appender; +import org.apache.log4j.Logger; +import org.mortbay.jetty.NCSARequestLog; +import org.mortbay.jetty.RequestLog; + +/** + * RequestLog object for use with Http + */ +public class HttpRequestLog { + + public static final Log LOG = LogFactory.getLog(HttpRequestLog.class); + private static final HashMap serverToComponent; + + static { + serverToComponent = new HashMap(); + serverToComponent.put("cluster", "resourcemanager"); + serverToComponent.put("hdfs", "namenode"); + serverToComponent.put("node", "nodemanager"); + } + + public static RequestLog getRequestLog(String name) { + + String lookup = serverToComponent.get(name); + if (lookup != null) { + name = lookup; + } + String loggerName = "http.requests." + name; + String appenderName = name + "requestlog"; + Log logger = LogFactory.getLog(loggerName); + + if (logger instanceof Log4JLogger) { + Log4JLogger httpLog4JLog = (Log4JLogger)logger; + Logger httpLogger = httpLog4JLog.getLogger(); + Appender appender = null; + + try { + appender = httpLogger.getAppender(appenderName); + } catch (LogConfigurationException e) { + LOG.warn("Http request log for " + loggerName + + " could not be created"); + throw e; + } + + if (appender == null) { + LOG.info("Http request log for " + loggerName + + " is not defined"); + return null; + } + + if (appender instanceof HttpRequestLogAppender) { + HttpRequestLogAppender requestLogAppender + = (HttpRequestLogAppender)appender; + NCSARequestLog requestLog = new NCSARequestLog(); + requestLog.setFilename(requestLogAppender.getFilename()); + requestLog.setRetainDays(requestLogAppender.getRetainDays()); + return requestLog; + } + else { + LOG.warn("Jetty request log for " + loggerName + + " was of the wrong class"); + return null; + } + } + else { + LOG.warn("Jetty request log can only be enabled using Log4j"); + return null; + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLogAppender.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLogAppender.java new file mode 100644 index 00000000000..eda1d1fee40 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLogAppender.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.http; + +import org.apache.log4j.spi.LoggingEvent; +import org.apache.log4j.AppenderSkeleton; + +/** + * Log4j Appender adapter for HttpRequestLog + */ +public class HttpRequestLogAppender extends AppenderSkeleton { + + private String filename; + private int retainDays; + + public HttpRequestLogAppender() { + } + + public void setRetainDays(int retainDays) { + this.retainDays = retainDays; + } + + public int getRetainDays() { + return retainDays; + } + + public void setFilename(String filename) { + this.filename = filename; + } + + public String getFilename() { + return filename; + } + + @Override + public void append(LoggingEvent event) { + } + + @Override + public void close() { + } + + @Override + public boolean requiresLayout() { + return false; + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java index 50582065473..b28e59424ed 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java @@ -67,9 +67,12 @@ import org.mortbay.io.Buffer; import org.mortbay.jetty.Connector; import org.mortbay.jetty.Handler; import org.mortbay.jetty.MimeTypes; +import org.mortbay.jetty.RequestLog; import org.mortbay.jetty.Server; import org.mortbay.jetty.handler.ContextHandler; import org.mortbay.jetty.handler.ContextHandlerCollection; +import org.mortbay.jetty.handler.RequestLogHandler; +import org.mortbay.jetty.handler.HandlerCollection; import org.mortbay.jetty.nio.SelectChannelConnector; import org.mortbay.jetty.security.SslSocketConnector; import org.mortbay.jetty.servlet.Context; @@ -355,7 +358,18 @@ public class HttpServer implements FilterContainer { final String appDir = getWebAppsPath(name); ContextHandlerCollection contexts = new ContextHandlerCollection(); - webServer.setHandler(contexts); + RequestLog requestLog = HttpRequestLog.getRequestLog(name); + + if (requestLog != null) { + RequestLogHandler requestLogHandler = new RequestLogHandler(); + requestLogHandler.setRequestLog(requestLog); + HandlerCollection handlers = new HandlerCollection(); + handlers.setHandlers(new Handler[] {requestLogHandler, contexts}); + webServer.setHandler(handlers); + } + else { + webServer.setHandler(contexts); + } webAppContext = new WebAppContext(); webAppContext.setDisplayName(name); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java index 52d569d6e6d..ee86ebcdd0d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java @@ -101,8 +101,12 @@ public class HttpServerFunctionalTest extends Assert { String webapps = System.getProperty(TEST_BUILD_WEBAPPS, BUILD_WEBAPPS_DIR); File testWebappDir = new File(webapps + File.separatorChar + TEST); + try { if (!testWebappDir.exists()) { - fail("Test webapp dir " + testWebappDir + " missing"); + fail("Test webapp dir " + testWebappDir.getCanonicalPath() + " missing"); + } + } + catch (IOException e) { } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java new file mode 100644 index 00000000000..23e0d3e8a1e --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.http; + +import org.apache.log4j.Logger; +import org.junit.Test; +import org.mortbay.jetty.NCSARequestLog; +import org.mortbay.jetty.RequestLog; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +public class TestHttpRequestLog { + + @Test + public void testAppenderUndefined() { + RequestLog requestLog = HttpRequestLog.getRequestLog("test"); + assertNull("RequestLog should be null", requestLog); + } + + @Test + public void testAppenderDefined() { + HttpRequestLogAppender requestLogAppender = new HttpRequestLogAppender(); + requestLogAppender.setName("testrequestlog"); + Logger.getLogger("http.requests.test").addAppender(requestLogAppender); + RequestLog requestLog = HttpRequestLog.getRequestLog("test"); + Logger.getLogger("http.requests.test").removeAppender(requestLogAppender); + assertNotNull("RequestLog should not be null", requestLog); + assertEquals("Class mismatch", NCSARequestLog.class, requestLog.getClass()); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLogAppender.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLogAppender.java new file mode 100644 index 00000000000..e84bee06e6e --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLogAppender.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.http; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class TestHttpRequestLogAppender { + + @Test + public void testParameterPropagation() { + + HttpRequestLogAppender requestLogAppender = new HttpRequestLogAppender(); + requestLogAppender.setFilename("jetty-namenode-yyyy_mm_dd.log"); + requestLogAppender.setRetainDays(17); + assertEquals("Filename mismatch", "jetty-namenode-yyyy_mm_dd.log", + requestLogAppender.getFilename()); + assertEquals("Retain days mismatch", 17, + requestLogAppender.getRetainDays()); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerLifecycle.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerLifecycle.java index 27dd67f39c5..d7330e87140 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerLifecycle.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerLifecycle.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.http; +import org.apache.log4j.Logger; import org.junit.Test; public class TestHttpServerLifecycle extends HttpServerFunctionalTest { @@ -66,6 +67,27 @@ public class TestHttpServerLifecycle extends HttpServerFunctionalTest { stop(server); } + /** + * Test that the server with request logging enabled + * + * @throws Throwable on failure + */ + @Test + public void testStartedServerWithRequestLog() throws Throwable { + HttpRequestLogAppender requestLogAppender = new HttpRequestLogAppender(); + requestLogAppender.setName("httprequestlog"); + requestLogAppender.setFilename(System.getProperty("test.build.data", "/tmp/") + + "jetty-name-yyyy_mm_dd.log"); + Logger.getLogger(HttpServer.class.getName() + ".test").addAppender(requestLogAppender); + HttpServer server = null; + server = createTestServer(); + assertNotLive(server); + server.start(); + assertAlive(server); + stop(server); + Logger.getLogger(HttpServer.class.getName() + ".test").removeAppender(requestLogAppender); + } + /** * Assert that the result of {@link HttpServer#toString()} contains the specific text * @param server server to examine From efc1048ffe10695666cc70be83c2b51cab8cdf0a Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Fri, 6 Sep 2013 22:57:44 +0000 Subject: [PATCH 09/21] Revert HADOOP-9877 because of breakage reported in HADOOP-9912 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1520713 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 - .../java/org/apache/hadoop/fs/Globber.java | 56 +++---------------- .../fs/FileContextMainOperationsBaseTest.java | 15 ----- .../hadoop/fs/TestFsShellReturnCode.java | 20 ------- .../fs/TestHDFSFileContextMainOperations.java | 11 ---- 5 files changed, 8 insertions(+), 96 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 22ea0bee75c..bc130ff364e 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -357,8 +357,6 @@ Release 2.3.0 - UNRELEASED HADOOP-9865. FileContext#globStatus has a regression with respect to relative path. (Chuan Lin via Colin Patrick McCabe) - HADOOP-9877. Fix listing of snapshot directories in globStatus. - (Binglin Chang via Andrew Wang) HADOOP-9909. org.apache.hadoop.fs.Stat should permit other LANG. (Shinichi Yamashita via Andrew Wang) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java index b0bd8490715..bae801ef0a3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java @@ -62,18 +62,6 @@ class Globber { } } - private FileStatus getFileLinkStatus(Path path) { - try { - if (fs != null) { - return fs.getFileLinkStatus(path); - } else { - return fc.getFileLinkStatus(path); - } - } catch (IOException e) { - return null; - } - } - private FileStatus[] listStatus(Path path) { try { if (fs != null) { @@ -134,18 +122,6 @@ class Globber { return authority ; } - /** - * The glob filter builds a regexp per path component. If the component - * does not contain a shell metachar, then it falls back to appending the - * raw string to the list of built up paths. This raw path needs to have - * the quoting removed. Ie. convert all occurrences of "\X" to "X" - * @param name of the path component - * @return the unquoted path component - */ - private static String unquotePathComponent(String name) { - return name.replaceAll("\\\\(.)", "$1"); - } - public FileStatus[] glob() throws IOException { // First we get the scheme and authority of the pattern that was passed // in. @@ -210,30 +186,14 @@ class Globber { resolvedCandidate.isDirectory() == false) { continue; } - // For components without pattern, we get its FileStatus directly - // using getFileLinkStatus for two reasons: - // 1. It should be faster to only get FileStatus needed rather than - // get all children. - // 2. Some special filesystem directories (e.g. HDFS snapshot - // directories) are not returned by listStatus, but do exist if - // checked explicitly via getFileLinkStatus. - if (globFilter.hasPattern()) { - FileStatus[] children = listStatus(candidate.getPath()); - for (FileStatus child : children) { - // Set the child path based on the parent path. - // This keeps the symlinks in our path. - child.setPath(new Path(candidate.getPath(), - child.getPath().getName())); - if (globFilter.accept(child.getPath())) { - newCandidates.add(child); - } - } - } else { - Path p = new Path(candidate.getPath(), unquotePathComponent(component)); - FileStatus s = getFileLinkStatus(p); - if (s != null) { - s.setPath(p); - newCandidates.add(s); + FileStatus[] children = listStatus(candidate.getPath()); + for (FileStatus child : children) { + // Set the child path based on the parent path. + // This keeps the symlinks in our path. + child.setPath(new Path(candidate.getPath(), + child.getPath().getName())); + if (globFilter.accept(child.getPath())) { + newCandidates.add(child); } } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java index 877a491bf9b..354f7aabfd6 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java @@ -30,7 +30,6 @@ import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.permission.FsPermission; import org.junit.After; import org.junit.Assert; -import org.junit.Assume; import org.junit.Before; import org.junit.Test; @@ -633,20 +632,6 @@ public abstract class FileContextMainOperationsBaseTest { filteredPaths)); } - protected Path getHiddenPathForTest() { - return null; - } - - @Test - public void testGlobStatusFilterWithHiddenPathTrivialFilter() - throws Exception { - Path hidden = getHiddenPathForTest(); - Assume.assumeNotNull(hidden); - FileStatus[] filteredPaths = fc.util().globStatus(hidden, DEFAULT_FILTER); - Assert.assertNotNull(filteredPaths); - Assert.assertEquals(1, filteredPaths.length); - } - @Test public void testWriteReadAndDeleteEmptyFile() throws Exception { writeReadAndDelete(0); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java index 2fff29e38d4..dcc19df3d4e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java @@ -517,26 +517,6 @@ public class TestFsShellReturnCode { } return stat; } - - @Override - public FileStatus getFileLinkStatus(Path p) throws IOException { - String f = makeQualified(p).toString(); - FileStatus stat = super.getFileLinkStatus(p); - - stat.getPermission(); - if (owners.containsKey(f)) { - stat.setOwner("STUB-"+owners.get(f)); - } else { - stat.setOwner("REAL-"+stat.getOwner()); - } - if (groups.containsKey(f)) { - stat.setGroup("STUB-"+groups.get(f)); - } else { - stat.setGroup("REAL-"+stat.getGroup()); - } - return stat; - } - } static class MyFsShell extends FsShell { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java index 8f5f14db614..6388bdd9e7a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java @@ -59,9 +59,6 @@ public class TestHDFSFileContextMainOperations extends defaultWorkingDirectory = fc.makeQualified( new Path("/user/" + UserGroupInformation.getCurrentUser().getShortUserName())); fc.mkdir(defaultWorkingDirectory, FileContext.DEFAULT_PERM, true); - // Make defaultWorkingDirectory snapshottable to enable - // testGlobStatusFilterWithHiddenPathTrivialFilter - cluster.getFileSystem().allowSnapshot(defaultWorkingDirectory); } private static void restartCluster() throws IOException, LoginException { @@ -76,9 +73,6 @@ public class TestHDFSFileContextMainOperations extends defaultWorkingDirectory = fc.makeQualified( new Path("/user/" + UserGroupInformation.getCurrentUser().getShortUserName())); fc.mkdir(defaultWorkingDirectory, FileContext.DEFAULT_PERM, true); - // Make defaultWorkingDirectory snapshottable to enable - // testGlobStatusFilterWithHiddenPathTrivialFilter - cluster.getFileSystem().allowSnapshot(defaultWorkingDirectory); } @AfterClass @@ -98,11 +92,6 @@ public class TestHDFSFileContextMainOperations extends super.tearDown(); } - @Override - protected Path getHiddenPathForTest() { - return new Path(defaultWorkingDirectory, ".snapshot"); - } - @Override protected Path getDefaultWorkingDirectory() { return defaultWorkingDirectory; From f35983b8056b7bd9ac4685acabef53f4dd0e355e Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Sat, 7 Sep 2013 05:43:04 +0000 Subject: [PATCH 10/21] YARN-1107. Fixed a bug in ResourceManager because of which RM in secure mode fails to restart. Contributed by Omkar Vinit Joshi. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1520726 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../resourcemanager/ClientRMService.java | 9 +-- .../server/resourcemanager/RMContext.java | 10 +++ .../server/resourcemanager/RMContextImpl.java | 24 +++++++ .../resourcemanager/ResourceManager.java | 23 +++---- .../security/DelegationTokenRenewer.java | 67 ++++++++++++++----- .../server/resourcemanager/TestRMRestart.java | 38 ++++++++--- .../security/TestDelegationTokenRenewer.java | 35 +++++++++- 8 files changed, 160 insertions(+), 49 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 7fe2e0af20c..9501969ebf5 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -156,6 +156,9 @@ Release 2.1.1-beta - UNRELEASED need more than a node's total capability were incorrectly allocated on that node causing apps to hang. (Omkar Vinit Joshi via vinodkv) + YARN-1107. Fixed a bug in ResourceManager because of which RM in secure mode + fails to restart. (Omkar Vinit Joshi via vinodkv) + Release 2.1.0-beta - 2013-08-22 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index d2888e77da0..11248bad48c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -43,10 +43,10 @@ import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; @@ -78,7 +78,6 @@ import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.RPCUtil; @@ -88,7 +87,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstant import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; @@ -160,9 +158,6 @@ public class ClientRMService extends AbstractService implements this.server.start(); clientBindAddress = conf.updateConnectAddr(YarnConfiguration.RM_ADDRESS, server.getListenerAddress()); - // enable RM to short-circuit token operations directly to itself - RMDelegationTokenIdentifier.Renewer.setSecretManager( - rmDTSecretManager, clientBindAddress); super.serviceStart(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java index b29da141ed6..28101cc27ed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java @@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSec import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; +import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager; /** * Context of the ResourceManager. @@ -64,4 +65,13 @@ public interface RMContext { NMTokenSecretManagerInRM getNMTokenSecretManager(); ClientToAMTokenSecretManagerInRM getClientToAMTokenSecretManager(); + + void setClientRMService(ClientRMService clientRMService); + + ClientRMService getClientRMService(); + + RMDelegationTokenSecretManager getRMDelegationTokenSecretManager(); + + void setRMDelegationTokenSecretManager( + RMDelegationTokenSecretManager delegationTokenSecretManager); } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java index f40453b5bbe..d2592ed445c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java @@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSec import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; +import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager; import com.google.common.annotations.VisibleForTesting; @@ -61,6 +62,8 @@ public class RMContextImpl implements RMContext { private final RMContainerTokenSecretManager containerTokenSecretManager; private final NMTokenSecretManagerInRM nmTokenSecretManager; private final ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager; + private ClientRMService clientRMService; + private RMDelegationTokenSecretManager rmDelegationTokenSecretManager; public RMContextImpl(Dispatcher rmDispatcher, RMStateStore store, @@ -178,4 +181,25 @@ public class RMContextImpl implements RMContext { public void setStateStore(RMStateStore store) { stateStore = store; } + + @Override + public ClientRMService getClientRMService() { + return this.clientRMService; + } + + @Override + public void setClientRMService(ClientRMService clientRMService) { + this.clientRMService = clientRMService; + } + + @Override + public RMDelegationTokenSecretManager getRMDelegationTokenSecretManager() { + return this.rmDelegationTokenSecretManager; + } + + @Override + public void setRMDelegationTokenSecretManager( + RMDelegationTokenSecretManager delegationTokenSecretManager) { + this.rmDelegationTokenSecretManager = delegationTokenSecretManager; + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index c0b372a753c..ee418c1937c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -169,11 +169,6 @@ public class ResourceManager extends CompositeService implements Recoverable { AMLivelinessMonitor amFinishingMonitor = createAMLivelinessMonitor(); addService(amFinishingMonitor); - if (UserGroupInformation.isSecurityEnabled()) { - this.delegationTokenRenewer = createDelegationTokenRenewer(); - addService(delegationTokenRenewer); - } - this.containerTokenSecretManager = createContainerTokenSecretManager(conf); this.nmTokenSecretManager = createNMTokenSecretManager(conf); @@ -200,6 +195,10 @@ public class ResourceManager extends CompositeService implements Recoverable { ExitUtil.terminate(1, e); } + if (UserGroupInformation.isSecurityEnabled()) { + this.delegationTokenRenewer = createDelegationTokenRenewer(); + } + this.rmContext = new RMContextImpl(this.rmDispatcher, rmStore, this.containerAllocationExpirer, amLivelinessMonitor, @@ -260,7 +259,9 @@ public class ResourceManager extends CompositeService implements Recoverable { this.rmDispatcher.register(RMAppManagerEventType.class, this.rmAppManager); this.rmDTSecretManager = createRMDelegationTokenSecretManager(this.rmContext); + rmContext.setRMDelegationTokenSecretManager(this.rmDTSecretManager); clientRM = createClientRMService(); + rmContext.setClientRMService(clientRM); addService(clientRM); adminService = createAdminService(clientRM, masterService, resourceTracker); @@ -271,7 +272,10 @@ public class ResourceManager extends CompositeService implements Recoverable { this.applicationMasterLauncher); addService(applicationMasterLauncher); - + if (UserGroupInformation.isSecurityEnabled()) { + addService(delegationTokenRenewer); + delegationTokenRenewer.setRMContext(rmContext); + } new RMNMInfo(this.rmContext, this.scheduler); super.serviceInit(conf); @@ -620,13 +624,6 @@ public class ResourceManager extends CompositeService implements Recoverable { this.containerTokenSecretManager.start(); this.nmTokenSecretManager.start(); - // Explicitly start DTRenewer too in secure mode before kicking recovery as - // tokens will start getting added for renewal as part of the recovery - // process itself. - if (UserGroupInformation.isSecurityEnabled()) { - this.delegationTokenRenewer.start(); - } - RMStateStore rmStore = rmContext.getStateStore(); // The state store needs to start irrespective of recoveryEnabled as apps // need events to move to further states. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java index bad5eabd407..a58b9175f8e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java @@ -34,6 +34,7 @@ import java.util.TimerTask; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -47,6 +48,8 @@ import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import com.google.common.annotations.VisibleForTesting; @@ -64,6 +67,7 @@ public class DelegationTokenRenewer extends AbstractService { // global single timer (daemon) private Timer renewalTimer; + private RMContext rmContext; // delegation token canceler thread private DelegationTokenCancelThread dtCancelThread = @@ -80,6 +84,9 @@ public class DelegationTokenRenewer extends AbstractService { private long tokenRemovalDelayMs; private Thread delayedRemovalThread; + private boolean isServiceStarted = false; + private List pendingTokenForRenewal = + new ArrayList(); private boolean tokenKeepAliveEnabled; @@ -100,7 +107,6 @@ public class DelegationTokenRenewer extends AbstractService { @Override protected void serviceStart() throws Exception { - dtCancelThread.start(); renewalTimer = new Timer(true); if (tokenKeepAliveEnabled) { @@ -109,6 +115,15 @@ public class DelegationTokenRenewer extends AbstractService { "DelayedTokenCanceller"); delayedRemovalThread.start(); } + // enable RM to short-circuit token operations directly to itself + RMDelegationTokenIdentifier.Renewer.setSecretManager( + rmContext.getRMDelegationTokenSecretManager(), + rmContext.getClientRMService().getBindAddress()); + // Delegation token renewal is delayed until ClientRMService starts. As + // it is required to short circuit the token renewal calls. + isServiceStarted = true; + renewIfServiceIsStarted(pendingTokenForRenewal); + pendingTokenForRenewal.clear(); super.serviceStart(); } @@ -275,8 +290,8 @@ public class DelegationTokenRenewer extends AbstractService { * @throws IOException */ public void addApplication( - ApplicationId applicationId, Credentials ts, boolean shouldCancelAtEnd) - throws IOException { + ApplicationId applicationId, Credentials ts, boolean shouldCancelAtEnd) + throws IOException { if (ts == null) { return; //nothing to add } @@ -291,25 +306,40 @@ public class DelegationTokenRenewer extends AbstractService { // find tokens for renewal, but don't add timers until we know // all renewable tokens are valid - Set dtrs = new HashSet(); + // At RM restart it is safe to assume that all the previously added tokens + // are valid + List tokenList = + new ArrayList(); for(Token token : tokens) { - // first renew happens immediately if (token.isManaged()) { - DelegationTokenToRenew dtr = - new DelegationTokenToRenew(applicationId, token, getConfig(), now, - shouldCancelAtEnd); - renewToken(dtr); - dtrs.add(dtr); + tokenList.add(new DelegationTokenToRenew(applicationId, + token, getConfig(), now, shouldCancelAtEnd)); } } - for (DelegationTokenToRenew dtr : dtrs) { - addTokenToList(dtr); - setTimerForTokenRenewal(dtr); - if (LOG.isDebugEnabled()) { - LOG.debug("Registering token for renewal for:" + - " service = " + dtr.token.getService() + - " for appId = " + applicationId); + if (!tokenList.isEmpty()){ + renewIfServiceIsStarted(tokenList); + } + } + + protected void renewIfServiceIsStarted(List dtrs) + throws IOException { + if (isServiceStarted) { + // Renewing token and adding it to timer calls are separated purposefully + // If user provides incorrect token then it should not be added for + // renewal. + for (DelegationTokenToRenew dtr : dtrs) { + renewToken(dtr); } + for (DelegationTokenToRenew dtr : dtrs) { + addTokenToList(dtr); + setTimerForTokenRenewal(dtr); + if (LOG.isDebugEnabled()) { + LOG.debug("Registering token for renewal for:" + " service = " + + dtr.token.getService() + " for appId = " + dtr.applicationId); + } + } + } else { + pendingTokenForRenewal.addAll(dtrs); } } @@ -513,4 +543,7 @@ public class DelegationTokenRenewer extends AbstractService { } } + public void setRMContext(RMContext rmContext) { + this.rmContext = rmContext; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index c0f480bd1c4..fbf53267dff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.resourcemanager; import java.io.IOException; -import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.nio.ByteBuffer; @@ -35,6 +34,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.SaslRpcServer.AuthMethod; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; @@ -90,7 +90,7 @@ public class TestRMRestart { conf.set(YarnConfiguration.RECOVERY_ENABLED, "true"); conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); - rmAddr = new InetSocketAddress(InetAddress.getLocalHost(), 123); + rmAddr = new InetSocketAddress("localhost", 8032); } @Test (timeout=180000) @@ -592,7 +592,12 @@ public class TestRMRestart { @Test public void testRMDelegationTokenRestoredOnRMRestart() throws Exception { conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2); - + + conf.set( + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, + "kerberos"); + conf.set(YarnConfiguration.RM_ADDRESS, "localhost:8032"); + UserGroupInformation.setConfiguration(conf); MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.init(conf); RMState rmState = memStore.getState(); @@ -614,6 +619,8 @@ public class TestRMRestart { // request a token and add into credential GetDelegationTokenRequest request1 = GetDelegationTokenRequest.newInstance("renewer1"); + UserGroupInformation.getCurrentUser().setAuthenticationMethod( + AuthMethod.KERBEROS); GetDelegationTokenResponse response1 = rm1.getClientRMService().getDelegationToken(request1); org.apache.hadoop.yarn.api.records.Token delegationToken1 = @@ -644,7 +651,7 @@ public class TestRMRestart { rm1.getRMDTSecretManager().getAllTokens(); Assert.assertEquals(tokenIdentSet, allTokensRM1.keySet()); Assert.assertEquals(allTokensRM1, rmDTState); - + // assert sequence number is saved Assert.assertEquals( rm1.getRMDTSecretManager().getLatestDTSequenceNumber(), @@ -682,7 +689,7 @@ public class TestRMRestart { // assert master keys and tokens are populated back to DTSecretManager Map allTokensRM2 = rm2.getRMDTSecretManager().getAllTokens(); - Assert.assertEquals(allTokensRM1, allTokensRM2); + Assert.assertEquals(allTokensRM2.keySet(), allTokensRM1.keySet()); // rm2 has its own master keys when it starts, we use containsAll here Assert.assertTrue(rm2.getRMDTSecretManager().getAllMasterKeys() .containsAll(allKeysRM1)); @@ -735,15 +742,24 @@ public class TestRMRestart { } @Override - protected void doSecureLogin() throws IOException { - // Do nothing. + protected ClientRMService createClientRMService() { + return new ClientRMService(getRMContext(), getResourceScheduler(), + rmAppManager, applicationACLsManager, rmDTSecretManager){ + @Override + protected void serviceStart() throws Exception { + // do nothing + } + + @Override + protected void serviceStop() throws Exception { + //do nothing + } + }; } @Override - protected void serviceInit(Configuration conf) throws Exception { - super.serviceInit(conf); - RMDelegationTokenIdentifier.Renewer.setSecretManager( - this.getRMDTSecretManager(), rmAddr); + protected void doSecureLogin() throws IOException { + // Do nothing. } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java index 38580d5dab5..98e6ab0f1b7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java @@ -25,8 +25,10 @@ import static org.mockito.Matchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.io.IOException; +import java.net.InetSocketAddress; import java.net.URI; import java.net.URISyntaxException; import java.util.Collections; @@ -48,9 +50,12 @@ import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenRenewer; import org.apache.hadoop.security.token.delegation.DelegationKey; +import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.junit.After; import org.junit.Before; @@ -141,6 +146,13 @@ public class TestDelegationTokenRenewer { Renewer.reset(); delegationTokenRenewer = new DelegationTokenRenewer(); delegationTokenRenewer.init(conf); + RMContext mockContext = mock(RMContext.class); + ClientRMService mockClientRMService = mock(ClientRMService.class); + when(mockContext.getClientRMService()).thenReturn(mockClientRMService); + InetSocketAddress sockAddr = + InetSocketAddress.createUnresolved("localhost", 1234); + when(mockClientRMService.getBindAddress()).thenReturn(sockAddr); + delegationTokenRenewer.setRMContext(mockContext); delegationTokenRenewer.start(); } @@ -454,6 +466,13 @@ public class TestDelegationTokenRenewer { YarnConfiguration.RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS, 1000l); localDtr.init(lconf); + RMContext mockContext = mock(RMContext.class); + ClientRMService mockClientRMService = mock(ClientRMService.class); + when(mockContext.getClientRMService()).thenReturn(mockClientRMService); + InetSocketAddress sockAddr = + InetSocketAddress.createUnresolved("localhost", 1234); + when(mockClientRMService.getBindAddress()).thenReturn(sockAddr); + localDtr.setRMContext(mockContext); localDtr.start(); MyFS dfs = (MyFS)FileSystem.get(lconf); @@ -511,6 +530,13 @@ public class TestDelegationTokenRenewer { YarnConfiguration.RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS, 1000l); localDtr.init(lconf); + RMContext mockContext = mock(RMContext.class); + ClientRMService mockClientRMService = mock(ClientRMService.class); + when(mockContext.getClientRMService()).thenReturn(mockClientRMService); + InetSocketAddress sockAddr = + InetSocketAddress.createUnresolved("localhost", 1234); + when(mockClientRMService.getBindAddress()).thenReturn(sockAddr); + localDtr.setRMContext(mockContext); localDtr.start(); MyFS dfs = (MyFS)FileSystem.get(lconf); @@ -550,7 +576,7 @@ public class TestDelegationTokenRenewer { } catch (InvalidToken ite) {} } - @Test(timeout=2000) + @Test(timeout=20000) public void testConncurrentAddApplication() throws IOException, InterruptedException, BrokenBarrierException { final CyclicBarrier startBarrier = new CyclicBarrier(2); @@ -579,6 +605,13 @@ public class TestDelegationTokenRenewer { // fire up the renewer final DelegationTokenRenewer dtr = new DelegationTokenRenewer(); dtr.init(conf); + RMContext mockContext = mock(RMContext.class); + ClientRMService mockClientRMService = mock(ClientRMService.class); + when(mockContext.getClientRMService()).thenReturn(mockClientRMService); + InetSocketAddress sockAddr = + InetSocketAddress.createUnresolved("localhost", 1234); + when(mockClientRMService.getBindAddress()).thenReturn(sockAddr); + dtr.setRMContext(mockContext); dtr.start(); // submit a job that blocks during renewal From 03ae82a311c033318496111d3dbd77bbaaa990b6 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Sat, 7 Sep 2013 07:28:19 +0000 Subject: [PATCH 11/21] YARN-696. Changed RMWebservice apps call to take in multiple application states. Contributed by Trevor Lorimer. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1520736 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../resourcemanager/webapp/RMWebServices.java | 42 ++++- .../webapp/TestRMWebServicesApps.java | 167 +++++++++++++++++- .../src/site/apt/ResourceManagerRest.apt.vm | 5 +- 4 files changed, 203 insertions(+), 14 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 9501969ebf5..25607864512 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -89,6 +89,9 @@ Release 2.1.1-beta - UNRELEASED YARN-758. Augment MockNM to use multiple cores (Karthik Kambatla via Sandy Ryza) + YARN-696. Changed RMWebservice apps call to take in multiple application + states. (Trevor Lorimer via vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index a9b1523eeae..a5c2f441e05 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; import java.io.IOException; +import java.util.Arrays; import java.util.Collection; import java.util.EnumSet; import java.util.HashSet; @@ -231,6 +232,7 @@ public class RMWebServices { @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) public AppsInfo getApps(@Context HttpServletRequest hsr, @QueryParam("state") String stateQuery, + @QueryParam("states") Set statesQuery, @QueryParam("finalStatus") String finalStatusQuery, @QueryParam("user") String userQuery, @QueryParam("queue") String queueQuery, @@ -245,6 +247,7 @@ public class RMWebServices { boolean checkStart = false; boolean checkEnd = false; boolean checkAppTypes = false; + boolean checkAppStates = false; long countNum = 0; // set values suitable in case both of begin/end not specified @@ -321,6 +324,36 @@ public class RMWebServices { checkAppTypes = true; } + String allAppStates; + RMAppState[] stateArray = RMAppState.values(); + allAppStates = Arrays.toString(stateArray); + + Set appStates = new HashSet(); + // stateQuery is deprecated. + if (stateQuery != null && !stateQuery.isEmpty()) { + statesQuery.add(stateQuery); + } + if (!statesQuery.isEmpty()) { + for (String applicationState : statesQuery) { + if (applicationState != null && !applicationState.isEmpty()) { + String[] states = applicationState.split(","); + for (String state : states) { + try { + RMAppState.valueOf(state.trim()); + } catch (IllegalArgumentException iae) { + throw new BadRequestException( + "Invalid application-state " + state + + " specified. It should be one of " + allAppStates); + } + appStates.add(state.trim().toLowerCase()); + } + } + } + } + if (!appStates.isEmpty()) { + checkAppStates = true; + } + final ConcurrentMap apps = rm.getRMContext() .getRMApps(); AppsInfo allApps = new AppsInfo(); @@ -329,11 +362,10 @@ public class RMWebServices { if (checkCount && num == countNum) { break; } - if (stateQuery != null && !stateQuery.isEmpty()) { - RMAppState.valueOf(stateQuery); - if (!rmapp.getState().toString().equalsIgnoreCase(stateQuery)) { - continue; - } + + if (checkAppStates + && !appStates.contains(rmapp.getState().toString().toLowerCase())) { + continue; } if (finalStatusQuery != null && !finalStatusQuery.isEmpty()) { FinalApplicationStatus.valueOf(finalStatusQuery); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 52f72d8692e..5e7145283ce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -68,6 +68,7 @@ import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.ClientResponse.Status; import com.sun.jersey.api.client.UniformInterfaceException; import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.core.util.MultivaluedMapImpl; import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; import com.sun.jersey.test.framework.JerseyTest; import com.sun.jersey.test.framework.WebAppDescriptor; @@ -239,6 +240,122 @@ public class TestRMWebServicesApps extends JerseyTest { rm.stop(); } + @Test + public void testAppsQueryStates() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + rm.submitApp(1024); + RMApp killedApp = rm.submitApp(1024); + rm.killApp(killedApp.getApplicationId()); + + amNodeManager.nodeHeartbeat(true); + + WebResource r = resource(); + MultivaluedMapImpl params = new MultivaluedMapImpl(); + params.add("states", RMAppState.ACCEPTED.toString()); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + JSONArray array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + assertEquals("state not equal to ACCEPTED", "ACCEPTED", array + .getJSONObject(0).getString("state")); + + r = resource(); + params = new MultivaluedMapImpl(); + params.add("states", RMAppState.ACCEPTED.toString()); + params.add("states", RMAppState.KILLED.toString()); + response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 2, array.length()); + assertTrue("both app states of ACCEPTED and KILLED are not present", + (array.getJSONObject(0).getString("state").equals("ACCEPTED") && + array.getJSONObject(1).getString("state").equals("KILLED")) || + (array.getJSONObject(0).getString("state").equals("KILLED") && + array.getJSONObject(1).getString("state").equals("ACCEPTED"))); + + rm.stop(); + } + + @Test + public void testAppsQueryStatesComma() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + rm.submitApp(1024); + RMApp killedApp = rm.submitApp(1024); + rm.killApp(killedApp.getApplicationId()); + + amNodeManager.nodeHeartbeat(true); + + WebResource r = resource(); + MultivaluedMapImpl params = new MultivaluedMapImpl(); + params.add("states", RMAppState.ACCEPTED.toString()); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + JSONArray array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + assertEquals("state not equal to ACCEPTED", "ACCEPTED", array + .getJSONObject(0).getString("state")); + + r = resource(); + params = new MultivaluedMapImpl(); + params.add("states", RMAppState.ACCEPTED.toString() + "," + + RMAppState.KILLED.toString()); + response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 2, array.length()); + assertTrue("both app states of ACCEPTED and KILLED are not present", + (array.getJSONObject(0).getString("state").equals("ACCEPTED") && + array.getJSONObject(1).getString("state").equals("KILLED")) || + (array.getJSONObject(0).getString("state").equals("KILLED") && + array.getJSONObject(1).getString("state").equals("ACCEPTED"))); + + rm.stop(); + } + + @Test + public void testAppsQueryStatesNone() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + rm.submitApp(1024); + amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); + + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParam("states", RMAppState.RUNNING.toString()) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + assertEquals("apps is not null", JSONObject.NULL, json.get("apps")); + rm.stop(); + } + @Test public void testAppsQueryStateNone() throws JSONException, Exception { rm.start(); @@ -257,6 +374,43 @@ public class TestRMWebServicesApps extends JerseyTest { rm.stop(); } + @Test + public void testAppsQueryStatesInvalid() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + rm.submitApp(1024); + amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); + + try { + r.path("ws").path("v1").path("cluster").path("apps") + .queryParam("states", "INVALID_test") + .accept(MediaType.APPLICATION_JSON).get(JSONObject.class); + fail("should have thrown exception on invalid state query"); + } catch (UniformInterfaceException ue) { + ClientResponse response = ue.getResponse(); + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject msg = response.getEntity(JSONObject.class); + JSONObject exception = msg.getJSONObject("RemoteException"); + assertEquals("incorrect number of elements", 3, exception.length()); + String message = exception.getString("message"); + String type = exception.getString("exception"); + String classname = exception.getString("javaClassName"); + WebServicesTestUtils.checkStringContains( + "exception message", + "Invalid application-state INVALID_test", + message); + WebServicesTestUtils.checkStringMatch("exception type", + "BadRequestException", type); + WebServicesTestUtils.checkStringMatch("exception classname", + "org.apache.hadoop.yarn.webapp.BadRequestException", classname); + + } finally { + rm.stop(); + } + } + @Test public void testAppsQueryStateInvalid() throws JSONException, Exception { rm.start(); @@ -280,15 +434,14 @@ public class TestRMWebServicesApps extends JerseyTest { String message = exception.getString("message"); String type = exception.getString("exception"); String classname = exception.getString("javaClassName"); - WebServicesTestUtils - .checkStringContains( - "exception message", - "org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState.INVALID_test", - message); + WebServicesTestUtils.checkStringContains( + "exception message", + "Invalid application-state INVALID_test", + message); WebServicesTestUtils.checkStringMatch("exception type", - "IllegalArgumentException", type); + "BadRequestException", type); WebServicesTestUtils.checkStringMatch("exception classname", - "java.lang.IllegalArgumentException", classname); + "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } finally { rm.stop(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm index 5f0580128df..22011124f27 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm @@ -1107,10 +1107,11 @@ ResourceManager REST API's. ** Query Parameters Supported - Multiple paramters can be specified. The started and finished times have a begin and end parameter to allow you to specify ranges. For example, one could request all applications that started between 1:00am and 2:00pm on 12/19/2011 with startedTimeBegin=1324256400&startedTimeEnd=1324303200. If the Begin parameter is not specfied, it defaults to 0, and if the End parameter is not specified, it defaults to infinity. + Multiple parameters can be specified. The started and finished times have a begin and end parameter to allow you to specify ranges. For example, one could request all applications that started between 1:00am and 2:00pm on 12/19/2011 with startedTimeBegin=1324256400&startedTimeEnd=1324303200. If the Begin parameter is not specified, it defaults to 0, and if the End parameter is not specified, it defaults to infinity. ------ - * state - state of the application + * state [deprecated] - state of the application + * states - applications matching the given application states, specified as a comma-separated list. * finalStatus - the final status of the application - reported by the application itself * user - user name * queue - queue name From 40cf0068d8aae2c50c90e0661aaa1170a4c03e42 Mon Sep 17 00:00:00 2001 From: Ivan Mitic Date: Sun, 8 Sep 2013 19:51:46 +0000 Subject: [PATCH 12/21] HADOOP-9924. Addendum patch to address issue with an empty classpath entry on Windows. Contributed by Shanyu Zhao. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1520903 13f79535-47bb-0310-9956-ffa450edef68 --- .../src/main/java/org/apache/hadoop/fs/FileUtil.java | 3 +++ .../src/test/java/org/apache/hadoop/fs/TestFileUtil.java | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java index bb203422f39..7bb20dd4032 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java @@ -1239,6 +1239,9 @@ public class FileUtil { List classPathEntryList = new ArrayList( classPathEntries.length); for (String classPathEntry: classPathEntries) { + if (classPathEntry.length() == 0) { + continue; + } if (classPathEntry.endsWith("*")) { // Append all jars that match the wildcard Path globPath = new Path(classPathEntry).suffix("{.jar,.JAR}"); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java index 3877e83a9b2..f37bf4f0c04 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java @@ -757,7 +757,7 @@ public class TestFileUtil { String wildcardPath = tmp.getCanonicalPath() + File.separator + "*"; String nonExistentSubdir = tmp.getCanonicalPath() + Path.SEPARATOR + "subdir" + Path.SEPARATOR; - List classPaths = Arrays.asList("cp1.jar", "cp2.jar", wildcardPath, + List classPaths = Arrays.asList("", "cp1.jar", "cp2.jar", wildcardPath, "cp3.jar", nonExistentSubdir); String inputClassPath = StringUtils.join(File.pathSeparator, classPaths); String classPathJar = FileUtil.createJarWithClassPath(inputClassPath, @@ -776,6 +776,9 @@ public class TestFileUtil { Assert.assertNotNull(classPathAttr); List expectedClassPaths = new ArrayList(); for (String classPath: classPaths) { + if (classPath.length() == 0) { + continue; + } if (wildcardPath.equals(classPath)) { // add wildcard matches for (File wildcardMatch: wildcardMatches) { From 4f2bf68b73df98f1bb5f80e6e192bd03b935b03b Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Mon, 9 Sep 2013 04:54:57 +0000 Subject: [PATCH 13/21] HDFS-5170. BlockPlacementPolicyDefault uses the wrong classname when alerting to enable debug logging. (Andrew Wang) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1520961 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../server/blockmanagement/BlockPlacementPolicyDefault.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 3dbcde5d619..735dd2634e4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -289,6 +289,9 @@ Release 2.3.0 - UNRELEASED HDFS-5164. deleteSnapshot should check if OperationCategory.WRITE is possible before taking write lock. (Colin Patrick McCabe) + HDFS-5170. BlockPlacementPolicyDefault uses the wrong classname when + alerting to enable debug logging. (Andrew Wang) + Release 2.1.1-beta - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index af5aab420a3..fbb922351bf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -55,7 +55,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { private static final String enableDebugLogging = "For more information, please enable DEBUG log level on " - + LOG.getClass().getName(); + + BlockPlacementPolicy.class.getName(); protected boolean considerLoad; private boolean preferLocalNode = true; From b2af6c70245703aa6d2ad1beceacc2aa2fcde5c0 Mon Sep 17 00:00:00 2001 From: Devarajulu K Date: Mon, 9 Sep 2013 06:42:42 +0000 Subject: [PATCH 14/21] MAPREDUCE-5414. TestTaskAttempt fails in JDK7 with NPE. Contributed by Nemon Lou. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1520964 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../mapreduce/v2/app/job/impl/TestTaskAttempt.java | 12 ++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index e3e0f25e83b..13f0079fc3c 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -253,6 +253,9 @@ Release 2.1.1-beta - UNRELEASED MAPREDUCE-5475. MRClientService does not verify ACLs properly (jlowe) + MAPREDUCE-5414. TestTaskAttempt fails in JDK7 with NPE (Nemon Lou via + devaraj) + Release 2.1.0-beta - 2013-08-22 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java index 885e01106d5..1129c2fcfc4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java @@ -343,7 +343,7 @@ public class TestTaskAttempt{ TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1, splits, jobConf, taListener, - mock(Token.class), new Credentials(), + new Token(), new Credentials(), new SystemClock(), null); NodeId nid = NodeId.newInstance("127.0.0.1", 0); @@ -399,7 +399,7 @@ public class TestTaskAttempt{ TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1, splits, jobConf, taListener, - mock(Token.class), new Credentials(), + new Token(), new Credentials(), new SystemClock(), appCtx); NodeId nid = NodeId.newInstance("127.0.0.2", 0); @@ -456,7 +456,7 @@ public class TestTaskAttempt{ TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1, splits, jobConf, taListener, - mock(Token.class), new Credentials(), + new Token(), new Credentials(), new SystemClock(), appCtx); NodeId nid = NodeId.newInstance("127.0.0.1", 0); @@ -516,7 +516,7 @@ public class TestTaskAttempt{ TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1, splits, jobConf, taListener, - mock(Token.class), new Credentials(), + new Token(), new Credentials(), new SystemClock(), appCtx); NodeId nid = NodeId.newInstance("127.0.0.1", 0); @@ -582,7 +582,7 @@ public class TestTaskAttempt{ TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1, splits, jobConf, taListener, - mock(Token.class), new Credentials(), new SystemClock(), appCtx); + new Token(), new Credentials(), new SystemClock(), appCtx); NodeId nid = NodeId.newInstance("127.0.0.1", 0); ContainerId contId = ContainerId.newInstance(appAttemptId, 3); @@ -631,7 +631,7 @@ public class TestTaskAttempt{ TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1, splits, jobConf, taListener, - mock(Token.class), new Credentials(), new SystemClock(), appCtx); + new Token(), new Credentials(), new SystemClock(), appCtx); NodeId nid = NodeId.newInstance("127.0.0.1", 0); ContainerId contId = ContainerId.newInstance(appAttemptId, 3); From 1cd7b067f7aebda201541e309ba27fc28e0b16db Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Mon, 9 Sep 2013 11:07:49 +0000 Subject: [PATCH 15/21] YARN-1049. ContainerExistStatus should define a status for preempted containers. (tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1521036 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../yarn/api/records/ContainerExitStatus.java | 5 +++ .../scheduler/SchedulerUtils.java | 39 ++++++++++++++++--- .../scheduler/capacity/CapacityScheduler.java | 2 +- .../scheduler/fair/FairScheduler.java | 2 +- .../scheduler/TestSchedulerUtils.java | 20 ++++++++++ 6 files changed, 64 insertions(+), 7 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 25607864512..c20d715a3e4 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -162,6 +162,9 @@ Release 2.1.1-beta - UNRELEASED YARN-1107. Fixed a bug in ResourceManager because of which RM in secure mode fails to restart. (Omkar Vinit Joshi via vinodkv) + YARN-1049. ContainerExistStatus should define a status for preempted + containers. (tucu) + Release 2.1.0-beta - 2013-08-22 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerExitStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerExitStatus.java index 76cb6c19581..71225787aeb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerExitStatus.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerExitStatus.java @@ -41,4 +41,9 @@ public class ContainerExitStatus { * threshold number of the nodemanager-log-directories become bad. */ public static final int DISKS_FAILED = -101; + + /** + * Containers preempted by the framework. + */ + public static final int PREEMPTED = -102; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java index b0cb25e5c75..ef71dccf510 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java @@ -63,7 +63,37 @@ public class SchedulerUtils { public static final String UNRESERVED_CONTAINER = "Container reservation no longer required."; - + + /** + * Utility to create a {@link ContainerStatus} during exceptional + * circumstances. + * + * @param containerId {@link ContainerId} of returned/released/lost container. + * @param diagnostics diagnostic message + * @return ContainerStatus for an returned/released/lost + * container + */ + public static ContainerStatus createAbnormalContainerStatus( + ContainerId containerId, String diagnostics) { + return createAbnormalContainerStatus(containerId, + ContainerExitStatus.ABORTED, diagnostics); + } + + /** + * Utility to create a {@link ContainerStatus} during exceptional + * circumstances. + * + * @param containerId {@link ContainerId} of returned/released/lost container. + * @param diagnostics diagnostic message + * @return ContainerStatus for an returned/released/lost + * container + */ + public static ContainerStatus createPreemptedContainerStatus( + ContainerId containerId, String diagnostics) { + return createAbnormalContainerStatus(containerId, + ContainerExitStatus.PREEMPTED, diagnostics); + } + /** * Utility to create a {@link ContainerStatus} during exceptional * circumstances. @@ -73,14 +103,13 @@ public class SchedulerUtils { * @return ContainerStatus for an returned/released/lost * container */ - public static ContainerStatus createAbnormalContainerStatus( - ContainerId containerId, String diagnostics) { + private static ContainerStatus createAbnormalContainerStatus( + ContainerId containerId, int exitStatus, String diagnostics) { ContainerStatus containerStatus = recordFactory.newRecordInstance(ContainerStatus.class); containerStatus.setContainerId(containerId); containerStatus.setDiagnostics(diagnostics); - containerStatus.setExitStatus( - ContainerExitStatus.ABORTED); + containerStatus.setExitStatus(exitStatus); containerStatus.setState(ContainerState.COMPLETE); return containerStatus; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 2efb9ad6719..bbf7f5cf699 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -904,7 +904,7 @@ public class CapacityScheduler LOG.debug("KILL_CONTAINER: container" + cont.toString()); } completedContainer(cont, - SchedulerUtils.createAbnormalContainerStatus( + SchedulerUtils.createPreemptedContainerStatus( cont.getContainerId(),"Container being forcibly preempted:" + cont.getContainerId()), RMContainerEventType.KILL); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 7f315781ef9..72000e91fc3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -444,7 +444,7 @@ public class FairScheduler implements ResourceScheduler { // proceed with kill if (time + waitTimeBeforeKill < clock.getTime()) { ContainerStatus status = - SchedulerUtils.createAbnormalContainerStatus( + SchedulerUtils.createPreemptedContainerStatus( container.getContainerId(), SchedulerUtils.PREEMPTED_CONTAINER); // TODO: Not sure if this ever actually adds this to the list of cleanup diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java index 9661057d133..9969db5a5e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java @@ -41,6 +41,10 @@ import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerExitStatus; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; @@ -358,4 +362,20 @@ public class TestSchedulerUtils { Priority low = Priority.newInstance(2); assertTrue(high.compareTo(low) > 0); } + + @Test + public void testCreateAbnormalContainerStatus() { + ContainerStatus cd = SchedulerUtils.createAbnormalContainerStatus( + ContainerId.newInstance(ApplicationAttemptId.newInstance( + ApplicationId.newInstance(System.currentTimeMillis(), 1), 1), 1), "x"); + Assert.assertEquals(ContainerExitStatus.ABORTED, cd.getExitStatus()); + } + + @Test + public void testCreatePreemptedContainerStatus() { + ContainerStatus cd = SchedulerUtils.createPreemptedContainerStatus( + ContainerId.newInstance(ApplicationAttemptId.newInstance( + ApplicationId.newInstance(System.currentTimeMillis(), 1), 1), 1), "x"); + Assert.assertEquals(ContainerExitStatus.PREEMPTED, cd.getExitStatus()); + } } From f2e0a125f44e2a529ac2ff74feb655741fed56ba Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Mon, 9 Sep 2013 11:12:45 +0000 Subject: [PATCH 16/21] YARN-1144. Unmanaged AMs registering a tracking URI should not be proxy-fied. (tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1521039 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../rmapp/attempt/RMAppAttemptImpl.java | 5 +- .../rmapp/attempt/TestRMAppAttemptImpl.java | 77 +++++++++++++++++++ .../attempt/TestRMAppAttemptTransitions.java | 56 ++++++++------ 4 files changed, 116 insertions(+), 25 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImpl.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index c20d715a3e4..17e53641ff0 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -165,6 +165,9 @@ Release 2.1.1-beta - UNRELEASED YARN-1049. ContainerExistStatus should define a status for preempted containers. (tucu) + YARN-1144. Unmanaged AMs registering a tracking URI should not be + proxy-fied. (tucu) + Release 2.1.0-beta - 2013-08-22 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 00397cfa650..51e6dc9f10d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -440,7 +440,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { public String getTrackingUrl() { this.readLock.lock(); try { - return this.proxiedTrackingUrl; + return (getSubmissionContext().getUnmanagedAM()) ? + this.origTrackingUrl : this.proxiedTrackingUrl; } finally { this.readLock.unlock(); } @@ -961,7 +962,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { } } - private static final class AMRegisteredTransition extends BaseTransition { + static final class AMRegisteredTransition extends BaseTransition { @Override public void transition(RMAppAttemptImpl appAttempt, RMAppAttemptEvent event) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImpl.java new file mode 100644 index 00000000000..e69d867e0f4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImpl.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt; + +import junit.framework.Assert; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptRegistrationEvent; + +import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils; +import org.junit.Test; +import org.mockito.Mockito; + +public class TestRMAppAttemptImpl { + + private void testTrackingUrl(String url, boolean unmanaged) { + ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance + (ApplicationId.newInstance(1, 2), 1); + EventHandler handler = Mockito.mock(EventHandler.class); + Dispatcher dispatcher = Mockito.mock(Dispatcher.class); + Mockito.when(dispatcher.getEventHandler()).thenReturn(handler); + RMContext rmContext = Mockito.mock(RMContext.class); + Mockito.when(rmContext.getDispatcher()).thenReturn(dispatcher); + + ApplicationSubmissionContext appContext = + Mockito.mock(ApplicationSubmissionContext.class); + Mockito.when(appContext.getUnmanagedAM()).thenReturn(unmanaged); + + RMAppAttemptImpl attempt = new RMAppAttemptImpl(attemptId, rmContext, null, + null, appContext, new YarnConfiguration(), null); + RMAppAttemptRegistrationEvent event = + Mockito.mock(RMAppAttemptRegistrationEvent.class); + Mockito.when(event.getHost()).thenReturn("h"); + Mockito.when(event.getRpcport()).thenReturn(0); + Mockito.when(event.getTrackingurl()).thenReturn(url); + new RMAppAttemptImpl.AMRegisteredTransition().transition(attempt, event); + if (unmanaged) { + Assert.assertEquals(url, attempt.getTrackingUrl()); + } else { + Assert.assertNotSame(url, attempt.getTrackingUrl()); + Assert.assertTrue(attempt.getTrackingUrl().contains( + ProxyUriUtils.PROXY_SERVLET_NAME)); + Assert.assertTrue(attempt.getTrackingUrl().contains( + attemptId.getApplicationId().toString())); + } + } + + @Test + public void testTrackingUrlUnmanagedAM() { + testTrackingUrl("http://foo:8000/x", true); + } + + @Test + public void testTrackingUrlManagedAM() { + testTrackingUrl("bar:8000/x", false); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index 5261d077d5c..2290284eaa8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -408,16 +408,19 @@ public class TestRMAppAttemptTransitions { * {@link RMAppAttemptState#RUNNING} */ private void testAppAttemptRunningState(Container container, - String host, int rpcPort, String trackingUrl) { + String host, int rpcPort, String trackingUrl, boolean unmanagedAM) { assertEquals(RMAppAttemptState.RUNNING, applicationAttempt.getAppAttemptState()); assertEquals(container, applicationAttempt.getMasterContainer()); assertEquals(host, applicationAttempt.getHost()); assertEquals(rpcPort, applicationAttempt.getRpcPort()); assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl()); - assertEquals(getProxyUrl(applicationAttempt), - applicationAttempt.getTrackingUrl()); - + if (unmanagedAM) { + assertEquals("oldtrackingurl", applicationAttempt.getTrackingUrl()); + } else { + assertEquals(getProxyUrl(applicationAttempt), + applicationAttempt.getTrackingUrl()); + } // TODO - need to add more checks relevant to this state } @@ -446,13 +449,18 @@ public class TestRMAppAttemptTransitions { FinalApplicationStatus finalStatus, String trackingUrl, String diagnostics, - int finishedContainerCount) { + int finishedContainerCount, boolean unmanagedAM) { assertEquals(RMAppAttemptState.FINISHED, applicationAttempt.getAppAttemptState()); assertEquals(diagnostics, applicationAttempt.getDiagnostics()); assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl()); - assertEquals(getProxyUrl(applicationAttempt), - applicationAttempt.getTrackingUrl()); + if (unmanagedAM) { + assertEquals("mytrackingurl", applicationAttempt.getTrackingUrl()); + + } else { + assertEquals(getProxyUrl(applicationAttempt), + applicationAttempt.getTrackingUrl()); + } assertEquals(finishedContainerCount, applicationAttempt .getJustFinishedContainers().size()); assertEquals(container, applicationAttempt.getMasterContainer()); @@ -535,13 +543,14 @@ public class TestRMAppAttemptTransitions { private void runApplicationAttempt(Container container, String host, int rpcPort, - String trackingUrl) { + String trackingUrl, boolean unmanagedAM) { applicationAttempt.handle( new RMAppAttemptRegistrationEvent( applicationAttempt.getAppAttemptId(), host, rpcPort, trackingUrl)); - testAppAttemptRunningState(container, host, rpcPort, trackingUrl); + testAppAttemptRunningState(container, host, rpcPort, trackingUrl, + unmanagedAM); } private void unregisterApplicationAttempt(Container container, @@ -567,7 +576,7 @@ public class TestRMAppAttemptTransitions { applicationAttempt.getAppAttemptId()); // launch AM - runApplicationAttempt(null, "host", 8042, "oldtrackingurl"); + runApplicationAttempt(null, "host", 8042, "oldtrackingurl", true); // complete a container applicationAttempt.handle(new RMAppAttemptContainerAcquiredEvent( @@ -581,7 +590,8 @@ public class TestRMAppAttemptTransitions { applicationAttempt.handle(new RMAppAttemptUnregistrationEvent( applicationAttempt.getAppAttemptId(), trackingUrl, finalStatus, diagnostics)); - testAppAttemptFinishedState(null, finalStatus, trackingUrl, diagnostics, 1); + testAppAttemptFinishedState(null, finalStatus, trackingUrl, diagnostics, 1, + true); } @Test @@ -690,7 +700,7 @@ public class TestRMAppAttemptTransitions { public void testRunningToFailed() { Container amContainer = allocateApplicationAttempt(); launchApplicationAttempt(amContainer); - runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl"); + runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false); String containerDiagMsg = "some error"; int exitCode = 123; ContainerStatus cs = BuilderUtils.newContainerStatus(amContainer.getId(), @@ -713,7 +723,7 @@ public class TestRMAppAttemptTransitions { public void testRunningToKilled() { Container amContainer = allocateApplicationAttempt(); launchApplicationAttempt(amContainer); - runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl"); + runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false); applicationAttempt.handle( new RMAppAttemptEvent( applicationAttempt.getAppAttemptId(), @@ -751,7 +761,7 @@ public class TestRMAppAttemptTransitions { public void testRunningExpire() { Container amContainer = allocateApplicationAttempt(); launchApplicationAttempt(amContainer); - runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl"); + runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false); applicationAttempt.handle(new RMAppAttemptEvent( applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.EXPIRE)); assertEquals(RMAppAttemptState.FAILED, @@ -769,7 +779,7 @@ public class TestRMAppAttemptTransitions { public void testUnregisterToKilledFinishing() { Container amContainer = allocateApplicationAttempt(); launchApplicationAttempt(amContainer); - runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl"); + runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false); unregisterApplicationAttempt(amContainer, FinalApplicationStatus.KILLED, "newtrackingurl", "Killed by user"); @@ -780,14 +790,14 @@ public class TestRMAppAttemptTransitions { public void testNoTrackingUrl() { Container amContainer = allocateApplicationAttempt(); launchApplicationAttempt(amContainer); - runApplicationAttempt(amContainer, "host", 8042, ""); + runApplicationAttempt(amContainer, "host", 8042, "", false); } @Test public void testUnregisterToSuccessfulFinishing() { Container amContainer = allocateApplicationAttempt(); launchApplicationAttempt(amContainer); - runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl"); + runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false); unregisterApplicationAttempt(amContainer, FinalApplicationStatus.SUCCEEDED, "mytrackingurl", "Successful"); } @@ -796,7 +806,7 @@ public class TestRMAppAttemptTransitions { public void testFinishingKill() { Container amContainer = allocateApplicationAttempt(); launchApplicationAttempt(amContainer); - runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl"); + runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false); FinalApplicationStatus finalStatus = FinalApplicationStatus.FAILED; String trackingUrl = "newtrackingurl"; String diagnostics = "Job failed"; @@ -814,7 +824,7 @@ public class TestRMAppAttemptTransitions { public void testFinishingExpire() { Container amContainer = allocateApplicationAttempt(); launchApplicationAttempt(amContainer); - runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl"); + runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false); FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED; String trackingUrl = "mytrackingurl"; String diagnostics = "Successful"; @@ -825,14 +835,14 @@ public class TestRMAppAttemptTransitions { applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.EXPIRE)); testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl, - diagnostics, 0); + diagnostics, 0, false); } @Test public void testFinishingToFinishing() { Container amContainer = allocateApplicationAttempt(); launchApplicationAttempt(amContainer); - runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl"); + runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false); FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED; String trackingUrl = "mytrackingurl"; String diagnostics = "Successful"; @@ -854,7 +864,7 @@ public class TestRMAppAttemptTransitions { public void testSuccessfulFinishingToFinished() { Container amContainer = allocateApplicationAttempt(); launchApplicationAttempt(amContainer); - runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl"); + runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false); FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED; String trackingUrl = "mytrackingurl"; String diagnostics = "Successful"; @@ -866,7 +876,7 @@ public class TestRMAppAttemptTransitions { BuilderUtils.newContainerStatus(amContainer.getId(), ContainerState.COMPLETE, "", 0))); testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl, - diagnostics, 0); + diagnostics, 0, false); } private void verifyTokenCount(ApplicationAttemptId appAttemptId, int count) { From 1a649aa51a3fe291b21bd2bb45d84fcd1806c521 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Mon, 9 Sep 2013 21:19:07 +0000 Subject: [PATCH 17/21] YARN-1152. Fixed a bug in ResourceManager that was causing clients to get invalid client token key errors when an appliation is about to finish. Contributed by Jason Lowe. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1521292 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 4 ++ .../resourcemanager/rmapp/RMAppImpl.java | 22 +++--- .../rmapp/attempt/RMAppAttempt.java | 8 +++ .../rmapp/attempt/RMAppAttemptImpl.java | 22 ++++++ .../rmapp/TestRMAppTransitions.java | 68 +++++++++++++++++-- .../attempt/TestRMAppAttemptTransitions.java | 31 +++++++++ 6 files changed, 138 insertions(+), 17 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 17e53641ff0..03747b3098f 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -168,6 +168,10 @@ Release 2.1.1-beta - UNRELEASED YARN-1144. Unmanaged AMs registering a tracking URI should not be proxy-fied. (tucu) + YARN-1152. Fixed a bug in ResourceManager that was causing clients to get + invalid client token key errors when an appliation is about to finish. + (Jason Lowe via vinodkv) + Release 2.1.0-beta - 2013-08-22 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index c69aed3473f..cbafffe04c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -432,18 +432,18 @@ public class RMAppImpl implements RMApp, Recoverable { currentApplicationAttemptId = this.currentAttempt.getAppAttemptId(); trackingUrl = this.currentAttempt.getTrackingUrl(); origTrackingUrl = this.currentAttempt.getOriginalTrackingUrl(); - if (UserGroupInformation.isSecurityEnabled() - && clientUserName != null) { + if (UserGroupInformation.isSecurityEnabled()) { + // get a token so the client can communicate with the app attempt + // NOTE: token may be unavailable if the attempt is not running Token attemptClientToAMToken = - new Token( - new ClientToAMTokenIdentifier( - currentApplicationAttemptId, clientUserName), - rmContext.getClientToAMTokenSecretManager()); - clientToAMToken = BuilderUtils.newClientToAMToken( - attemptClientToAMToken.getIdentifier(), - attemptClientToAMToken.getKind().toString(), - attemptClientToAMToken.getPassword(), - attemptClientToAMToken.getService().toString()); + this.currentAttempt.createClientToken(clientUserName); + if (attemptClientToAMToken != null) { + clientToAMToken = BuilderUtils.newClientToAMToken( + attemptClientToAMToken.getIdentifier(), + attemptClientToAMToken.getKind().toString(), + attemptClientToAMToken.getPassword(), + attemptClientToAMToken.getService().toString()); + } } host = this.currentAttempt.getHost(); rpcPort = this.currentAttempt.getRpcPort(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java index e9f064d648e..aa44c743ccf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java @@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; +import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; /** @@ -155,6 +156,13 @@ public interface RMAppAttempt extends EventHandler { */ SecretKey getClientTokenMasterKey(); + /** + * Create a token for authenticating a client connection to the app attempt + * @param clientName the name of the client requesting the token + * @return the token or null if the attempt is not running + */ + Token createClientToken(String clientName); + /** * Get application container and resource usage information. * @return an ApplicationResourceUsageReport object. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 51e6dc9f10d..0e1b2c8a53f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -61,6 +61,7 @@ import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; +import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent; @@ -89,6 +90,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppRepor import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils; import org.apache.hadoop.yarn.state.InvalidStateTransitonException; @@ -508,6 +510,26 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { return this.amrmToken; } + @Override + public Token createClientToken(String client) { + this.readLock.lock(); + + try { + Token token = null; + ClientToAMTokenSecretManagerInRM secretMgr = + this.rmContext.getClientToAMTokenSecretManager(); + if (client != null && + secretMgr.getMasterKey(this.applicationAttemptId) != null) { + token = new Token( + new ClientToAMTokenIdentifier(this.applicationAttemptId, client), + secretMgr); + } + return token; + } finally { + this.readLock.unlock(); + } + } + @Override public String getDiagnostics() { this.readLock.lock(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index d6bd3f6a0c1..2c19597de9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -19,14 +19,20 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp; import static org.mockito.Mockito.mock; +import static org.junit.Assume.assumeTrue; import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; import junit.framework.Assert; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; @@ -57,11 +63,16 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSe import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +@RunWith(value = Parameterized.class) public class TestRMAppTransitions { static final Log LOG = LogFactory.getLog(TestRMAppTransitions.class); + private boolean isSecurityEnabled; + private Configuration conf; private RMContext rmContext; private static int maxAppAttempts = YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS; @@ -132,10 +143,29 @@ public class TestRMAppTransitions { public void handle(SchedulerEvent event) { } } + + @Parameterized.Parameters + public static Collection getTestParameters() { + return Arrays.asList(new Object[][] { + { Boolean.FALSE }, + { Boolean.TRUE } + }); + } + + public TestRMAppTransitions(boolean isSecurityEnabled) { + this.isSecurityEnabled = isSecurityEnabled; + } @Before public void setUp() throws Exception { - Configuration conf = new Configuration(); + conf = new YarnConfiguration(); + AuthenticationMethod authMethod = AuthenticationMethod.SIMPLE; + if (isSecurityEnabled) { + authMethod = AuthenticationMethod.KERBEROS; + } + SecurityUtil.setAuthenticationMethod(authMethod, conf); + UserGroupInformation.setConfiguration(conf); + rmDispatcher = new DrainDispatcher(); ContainerAllocationExpirer containerAllocationExpirer = mock(ContainerAllocationExpirer.class); @@ -171,7 +201,6 @@ public class TestRMAppTransitions { String user = MockApps.newUserName(); String name = MockApps.newAppName(); String queue = MockApps.newQueue(); - Configuration conf = new YarnConfiguration(); // ensure max application attempts set to known value conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, maxAppAttempts); YarnScheduler scheduler = mock(YarnScheduler.class); @@ -191,6 +220,8 @@ public class TestRMAppTransitions { System.currentTimeMillis(), "YARN"); testAppStartState(applicationId, user, name, queue, application); + this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(), + application); return application; } @@ -488,8 +519,6 @@ public class TestRMAppTransitions { // SUBMITTED => KILLED event RMAppEventType.KILL RMAppEvent event = new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL); - this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(), - application); application.handle(event); rmDispatcher.await(); assertKilled(application); @@ -535,8 +564,6 @@ public class TestRMAppTransitions { // ACCEPTED => KILLED event RMAppEventType.KILL RMAppEvent event = new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL); - this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(), - application); application.handle(event); rmDispatcher.await(); assertKilled(application); @@ -731,4 +758,33 @@ public class TestRMAppTransitions { report = app.createAndGetApplicationReport("clientuser", true); Assert.assertNotNull(report.getApplicationResourceUsageReport()); } + + @Test + public void testClientTokens() throws Exception { + assumeTrue(isSecurityEnabled); + + RMApp app = createNewTestApp(null); + assertAppState(RMAppState.NEW, app); + ApplicationReport report = app.createAndGetApplicationReport(null, true); + Assert.assertNull(report.getClientToAMToken()); + report = app.createAndGetApplicationReport("clientuser", true); + Assert.assertNull(report.getClientToAMToken()); + + app = testCreateAppRunning(null); + rmDispatcher.await(); + assertAppState(RMAppState.RUNNING, app); + report = app.createAndGetApplicationReport(null, true); + Assert.assertNull(report.getClientToAMToken()); + report = app.createAndGetApplicationReport("clientuser", true); + Assert.assertNotNull(report.getClientToAMToken()); + + // kill the app attempt and verify client token is unavailable + app.handle(new RMAppEvent(app.getApplicationId(), RMAppEventType.KILL)); + rmDispatcher.await(); + assertAppAndAttemptKilled(app); + report = app.createAndGetApplicationReport(null, true); + Assert.assertNull(report.getClientToAMToken()); + report = app.createAndGetApplicationReport("clientuser", true); + Assert.assertNull(report.getClientToAMToken()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index 2290284eaa8..39c633761d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -30,13 +30,17 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.mockito.Mockito.spy; +import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -85,7 +89,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManag import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +@RunWith(value = Parameterized.class) public class TestRMAppAttemptTransitions { private static final Log LOG = @@ -95,6 +102,7 @@ public class TestRMAppAttemptTransitions { private static final String RM_WEBAPP_ADDR = YarnConfiguration.getRMWebAppHostAndPort(new Configuration()); + private boolean isSecurityEnabled; private RMContext rmContext; private YarnScheduler scheduler; private ApplicationMasterService masterService; @@ -162,8 +170,26 @@ public class TestRMAppAttemptTransitions { private ApplicationSubmissionContext submissionContext = null; private boolean unmanagedAM; + @Parameterized.Parameters + public static Collection getTestParameters() { + return Arrays.asList(new Object[][] { + { Boolean.FALSE }, + { Boolean.TRUE } + }); + } + + public TestRMAppAttemptTransitions(Boolean isSecurityEnabled) { + this.isSecurityEnabled = isSecurityEnabled; + } + @Before public void setUp() throws Exception { + AuthenticationMethod authMethod = AuthenticationMethod.SIMPLE; + if (isSecurityEnabled) { + authMethod = AuthenticationMethod.KERBEROS; + } + SecurityUtil.setAuthenticationMethod(authMethod, conf); + UserGroupInformation.setConfiguration(conf); InlineDispatcher rmDispatcher = new InlineDispatcher(); ContainerAllocationExpirer containerAllocationExpirer = @@ -270,7 +296,9 @@ public class TestRMAppAttemptTransitions { if (UserGroupInformation.isSecurityEnabled()) { verify(clientToAMTokenManager).registerApplication( applicationAttempt.getAppAttemptId()); + assertNotNull(applicationAttempt.createClientToken("some client")); } + assertNull(applicationAttempt.createClientToken(null)); assertNotNull(applicationAttempt.getAMRMToken()); // Check events verify(masterService). @@ -883,6 +911,9 @@ public class TestRMAppAttemptTransitions { verify(amRMTokenManager, times(count)).applicationMasterFinished(appAttemptId); if (UserGroupInformation.isSecurityEnabled()) { verify(clientToAMTokenManager, times(count)).unRegisterApplication(appAttemptId); + if (count > 0) { + assertNull(applicationAttempt.createClientToken("client")); + } } } } From 0f91d8485a6cd96153ad35e9babab248b20f53dc Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Mon, 9 Sep 2013 21:48:09 +0000 Subject: [PATCH 18/21] YARN-910. Augmented auxiliary services to listen for container starts and completions in addition to application events. Contributed by Alejandro Abdelnur. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1521298 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 4 + .../yarn/server/api/AuxiliaryService.java | 25 ++++++- .../yarn/server/api/ContainerContext.java | 75 +++++++++++++++++++ .../api/ContainerInitializationContext.java | 44 +++++++++++ .../api/ContainerTerminationContext.java | 44 +++++++++++ .../containermanager/AuxServices.java | 18 ++++- .../containermanager/AuxServicesEvent.java | 19 +++++ .../AuxServicesEventType.java | 4 +- .../container/ContainerImpl.java | 11 +++ .../containermanager/TestAuxServices.java | 61 ++++++++++++++- 10 files changed, 300 insertions(+), 5 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerContext.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerInitializationContext.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerTerminationContext.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 03747b3098f..ff351494297 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -28,6 +28,10 @@ Release 2.3.0 - UNRELEASED YARN-905. Add state filters to nodes CLI (Wei Yan via Sandy Ryza) + YARN-910. Augmented auxiliary services to listen for container starts and + completions in addition to application events. (Alejandro Abdelnur via + vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java index 275f2a91038..58b06e274a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java @@ -79,4 +79,27 @@ public abstract class AuxiliaryService extends AbstractService { * applications. */ public abstract ByteBuffer getMetaData(); -} \ No newline at end of file + + /** + * A new container is started on this NodeManager. This is a signal to + * this {@link AuxiliaryService} about the container initialization. + * This method is called when the NodeManager receives the container launch + * command from the ApplicationMaster and before the container process is + * launched. + * + * @param initContainerContext context for the container's initialization + */ + public void initializeContainer(ContainerInitializationContext + initContainerContext) { + } + + /** + * A container is finishing on this NodeManager. This is a signal to this + * {@link AuxiliaryService} about the same. + * + * @param stopContainerContext context for the container termination + */ + public void stopContainer(ContainerTerminationContext stopContainerContext) { + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerContext.java new file mode 100644 index 00000000000..d13159b308f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerContext.java @@ -0,0 +1,75 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.api; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.Resource; + +/** + * Base context class for {@link AuxiliaryService} initializing and stopping a + * container. + */ +@Public +@Evolving +public class ContainerContext { + private final String user; + private final ContainerId containerId; + private final Resource resource; + + @Private + @Unstable + public ContainerContext(String user, ContainerId containerId, + Resource resource) { + this.user = user; + this.containerId = containerId; + this.resource = resource; + } + + /** + * Get user of the container being initialized or stopped. + * + * @return the user + */ + public String getUser() { + return user; + } + + /** + * Get {@link ContainerId} of the container being initialized or stopped. + * + * @return the container ID + */ + public ContainerId getContainerId() { + return containerId; + } + + /** + * Get {@link Resource} the resource capability allocated to the container + * being initialized or stopped. + * + * @return the resource capability. + */ + public Resource getResource() { + return resource; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerInitializationContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerInitializationContext.java new file mode 100644 index 00000000000..5b5bbda0c07 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerInitializationContext.java @@ -0,0 +1,44 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.api; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.Resource; + +/** + * Initialization context for {@link AuxiliaryService} when starting a + * container. + * + */ +@Public +@Evolving +public class ContainerInitializationContext extends ContainerContext { + + @Private + @Unstable + public ContainerInitializationContext(String user, ContainerId containerId, + Resource resource) { + super(user, containerId, resource); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerTerminationContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerTerminationContext.java new file mode 100644 index 00000000000..34ba73e2213 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerTerminationContext.java @@ -0,0 +1,44 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.api; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.Resource; + +/** + * Termination context for {@link AuxiliaryService} when stopping a + * container. + * + */ +@Public +@Evolving +public class ContainerTerminationContext extends ContainerContext { + + @Private + @Unstable + public ContainerTerminationContext(String user, ContainerId containerId, + Resource resource) { + super(user, containerId, resource); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java index 955ccbf19ea..13f43650a47 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java @@ -37,6 +37,8 @@ import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.api.ApplicationTerminationContext; import org.apache.hadoop.yarn.server.api.AuxiliaryService; import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext; +import org.apache.hadoop.yarn.server.api.ContainerInitializationContext; +import org.apache.hadoop.yarn.server.api.ContainerTerminationContext; public class AuxServices extends AbstractService implements ServiceStateChangeListener, EventHandler { @@ -178,7 +180,21 @@ public class AuxServices extends AbstractService .getApplicationID())); } break; - default: + case CONTAINER_INIT: + for (AuxiliaryService serv : serviceMap.values()) { + serv.initializeContainer(new ContainerInitializationContext( + event.getUser(), event.getContainer().getContainerId(), + event.getContainer().getResource())); + } + break; + case CONTAINER_STOP: + for (AuxiliaryService serv : serviceMap.values()) { + serv.stopContainer(new ContainerTerminationContext( + event.getUser(), event.getContainer().getContainerId(), + event.getContainer().getResource())); + } + break; + default: throw new RuntimeException("Unknown type: " + event.getType()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEvent.java index 4b9c93157b2..1e5a9a737f5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEvent.java @@ -21,7 +21,10 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager; import java.nio.ByteBuffer; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.event.AbstractEvent; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container + .Container; public class AuxServicesEvent extends AbstractEvent { @@ -29,18 +32,30 @@ public class AuxServicesEvent extends AbstractEvent { private final String serviceId; private final ByteBuffer serviceData; private final ApplicationId appId; + private final Container container; public AuxServicesEvent(AuxServicesEventType eventType, ApplicationId appId) { this(eventType, null, appId, null, null); } + public AuxServicesEvent(AuxServicesEventType eventType, Container container) { + this(eventType, null, container.getContainerId().getApplicationAttemptId() + .getApplicationId(), null, null, container); + } + public AuxServicesEvent(AuxServicesEventType eventType, String user, ApplicationId appId, String serviceId, ByteBuffer serviceData) { + this(eventType, user, appId, serviceId, serviceData, null); + } + public AuxServicesEvent(AuxServicesEventType eventType, String user, + ApplicationId appId, String serviceId, ByteBuffer serviceData, + Container container) { super(eventType); this.user = user; this.appId = appId; this.serviceId = serviceId; this.serviceData = serviceData; + this.container = container; } public String getServiceID() { @@ -59,4 +74,8 @@ public class AuxServicesEvent extends AbstractEvent { return appId; } + public Container getContainer() { + return container; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEventType.java index b8276b0ba14..45f3c8f2906 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEventType.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEventType.java @@ -20,5 +20,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager; public enum AuxServicesEventType { APPLICATION_INIT, - APPLICATION_STOP + APPLICATION_STOP, + CONTAINER_INIT, + CONTAINER_STOP } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index 54a2cbec56d..c2d32b57bee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -503,6 +503,9 @@ public class ContainerImpl implements Container { final ContainerLaunchContext ctxt = container.launchContext; container.metrics.initingContainer(); + container.dispatcher.getEventHandler().handle(new AuxServicesEvent + (AuxServicesEventType.CONTAINER_INIT, container)); + // Inform the AuxServices about the opaque serviceData Map csd = ctxt.getServiceData(); if (csd != null) { @@ -820,8 +823,16 @@ public class ContainerImpl implements Container { static class ContainerDoneTransition implements SingleArcTransition { @Override + @SuppressWarnings("unchecked") public void transition(ContainerImpl container, ContainerEvent event) { container.finished(); + //if the current state is NEW it means the CONTAINER_INIT was never + // sent for the event, thus no need to send the CONTAINER_STOP + if (container.getCurrentState() + != org.apache.hadoop.yarn.api.records.ContainerState.NEW) { + container.dispatcher.getEventHandler().handle(new AuxServicesEvent + (AuxServicesEventType.CONTAINER_STOP, container)); + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java index fb4b69a21f0..81f758ee063 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java @@ -22,6 +22,7 @@ import static org.apache.hadoop.service.Service.STATE.INITED; import static org.apache.hadoop.service.Service.STATE.STARTED; import static org.apache.hadoop.service.Service.STATE.STOPPED; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -34,11 +35,21 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.service.Service; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext; import org.apache.hadoop.yarn.server.api.ApplicationTerminationContext; import org.apache.hadoop.yarn.server.api.AuxiliaryService; +import org.apache.hadoop.yarn.server.api.ContainerInitializationContext; +import org.apache.hadoop.yarn.server.api.ContainerTerminationContext; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container + .Container; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container + .ContainerImpl; import org.junit.Test; public class TestAuxServices { @@ -52,8 +63,10 @@ public class TestAuxServices { private int remaining_stop; private ByteBuffer meta = null; private ArrayList stoppedApps; + private ContainerId containerId; + private Resource resource; - LightService(String name, char idef, int expected_appId) { + LightService(String name, char idef, int expected_appId) { this(name, idef, expected_appId, null); } LightService(String name, char idef, int expected_appId, ByteBuffer meta) { @@ -95,7 +108,22 @@ public class TestAuxServices { public ByteBuffer getMetaData() { return meta; } - } + + @Override + public void initializeContainer( + ContainerInitializationContext initContainerContext) { + containerId = initContainerContext.getContainerId(); + resource = initContainerContext.getResource(); + } + + @Override + public void stopContainer( + ContainerTerminationContext stopContainerContext) { + containerId = stopContainerContext.getContainerId(); + resource = stopContainerContext.getResource(); + } + + } static class ServiceA extends LightService { public ServiceA() { @@ -142,6 +170,35 @@ public class TestAuxServices { assertEquals("app not properly stopped", 1, appIds.size()); assertTrue("wrong app stopped", appIds.contains((Integer)66)); } + + for (AuxiliaryService serv : servs) { + assertNull(((LightService) serv).containerId); + assertNull(((LightService) serv).resource); + } + + + ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId1, 1); + ContainerTokenIdentifier cti = new ContainerTokenIdentifier( + ContainerId.newInstance(attemptId, 1), "", "", + Resource.newInstance(1, 1), 0,0,0); + Container container = new ContainerImpl(null, null, null, null, null, cti); + ContainerId containerId = container.getContainerId(); + Resource resource = container.getResource(); + event = new AuxServicesEvent(AuxServicesEventType.CONTAINER_INIT,container); + aux.handle(event); + for (AuxiliaryService serv : servs) { + assertEquals(containerId, ((LightService) serv).containerId); + assertEquals(resource, ((LightService) serv).resource); + ((LightService) serv).containerId = null; + ((LightService) serv).resource = null; + } + + event = new AuxServicesEvent(AuxServicesEventType.CONTAINER_STOP, container); + aux.handle(event); + for (AuxiliaryService serv : servs) { + assertEquals(containerId, ((LightService) serv).containerId); + assertEquals(resource, ((LightService) serv).resource); + } } @Test From 59b5490989fad4c8e80dd85c3419810cdc8332f7 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Tue, 10 Sep 2013 01:24:45 +0000 Subject: [PATCH 19/21] YARN-292. Fixed FifoScheduler and FairScheduler to make their applications data structures thread safe to avoid RM crashing with ArrayIndexOutOfBoundsException. Contributed by Zhijie Shen. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1521328 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 4 + .../rmapp/attempt/RMAppAttemptImpl.java | 6 +- .../scheduler/capacity/CapacityScheduler.java | 5 +- .../scheduler/fair/FairScheduler.java | 5 +- .../scheduler/fifo/FifoScheduler.java | 14 ++-- .../capacity/TestCapacityScheduler.java | 75 ++++++++++++++++++- .../scheduler/fair/TestFairScheduler.java | 9 +++ .../scheduler/fifo/TestFifoScheduler.java | 13 +++- 8 files changed, 117 insertions(+), 14 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index ff351494297..b211a18bbb7 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -176,6 +176,10 @@ Release 2.1.1-beta - UNRELEASED invalid client token key errors when an appliation is about to finish. (Jason Lowe via vinodkv) + YARN-292. Fixed FifoScheduler and FairScheduler to make their applications + data structures thread safe to avoid RM crashing with + ArrayIndexOutOfBoundsException. (Zhijie Shen via vinodkv) + Release 2.1.0-beta - 2013-08-22 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 0e1b2c8a53f..94a0f94b572 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -833,7 +833,11 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { Allocation amContainerAllocation = appAttempt.scheduler.allocate( appAttempt.applicationAttemptId, EMPTY_CONTAINER_REQUEST_LIST, EMPTY_CONTAINER_RELEASE_LIST, null, null); - + // There must be at least one container allocated, because a + // CONTAINER_ALLOCATED is emitted after an RMContainer is constructed, + // and is put in SchedulerApplication#newlyAllocatedContainers. Then, + // YarnScheduler#allocate will fetch it. + assert amContainerAllocation.getContainers().size() != 0; // Set the masterContainer appAttempt.setMasterContainer(amContainerAllocation.getContainers().get( 0)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index bbf7f5cf699..9edf420da00 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -80,6 +80,8 @@ import org.apache.hadoop.yarn.server.utils.Lock; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; +import com.google.common.annotations.VisibleForTesting; + @LimitedPrivate("yarn") @Evolving @SuppressWarnings("unchecked") @@ -179,7 +181,8 @@ public class CapacityScheduler private Resource minimumAllocation; private Resource maximumAllocation; - private Map applications = + @VisibleForTesting + protected Map applications = new ConcurrentHashMap(); private boolean initialized = false; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 72000e91fc3..73221042a76 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -37,7 +37,6 @@ import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -50,6 +49,7 @@ import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; @@ -154,8 +154,9 @@ public class FairScheduler implements ResourceScheduler { // This stores per-application scheduling information, indexed by // attempt ID's for fast lookup. + @VisibleForTesting protected Map applications = - new HashMap(); + new ConcurrentHashMap(); // Nodes in the cluster, indexed by NodeId private Map nodes = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 115d2089c34..bac013f365a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -25,8 +25,8 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentSkipListMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -36,12 +36,10 @@ import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; @@ -50,7 +48,9 @@ import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger; @@ -90,6 +90,8 @@ import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; +import com.google.common.annotations.VisibleForTesting; + @LimitedPrivate("yarn") @Evolving @SuppressWarnings("unchecked") @@ -113,8 +115,10 @@ public class FifoScheduler implements ResourceScheduler, Configurable { private Resource maximumAllocation; private boolean usePortForNodeName; - private Map applications - = new TreeMap(); + // Use ConcurrentSkipListMap because applications need to be ordered + @VisibleForTesting + protected Map applications + = new ConcurrentSkipListMap(); private ActiveUsersManager activeUsersManager; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index ec486d78ea0..38df24fa991 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -19,22 +19,27 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import java.io.IOException; +import java.lang.reflect.Constructor; import java.util.Comparator; import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import junit.framework.Assert; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.net.NetworkTopology; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueInfo; @@ -44,19 +49,24 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.resourcemanager.Application; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.Task; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; -import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; +import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; import org.junit.Before; @@ -525,4 +535,63 @@ public class TestCapacityScheduler { assertTrue(appComparator.compare(app2, app3) < 0); } + @Test + public void testConcurrentAccessOnApplications() throws Exception { + CapacityScheduler cs = new CapacityScheduler(); + verifyConcurrentAccessOnApplications( + cs.applications, FiCaSchedulerApp.class); + } + + public static + void verifyConcurrentAccessOnApplications( + final Map applications, Class clazz) + throws Exception { + final int size = 10000; + final ApplicationId appId = ApplicationId.newInstance(0, 0); + final Constructor ctor = clazz.getDeclaredConstructor( + ApplicationAttemptId.class, String.class, Queue.class, + ActiveUsersManager.class, RMContext.class); + + ApplicationAttemptId appAttemptId0 + = ApplicationAttemptId.newInstance(appId, 0); + applications.put(appAttemptId0, ctor.newInstance( + appAttemptId0, null, mock(Queue.class), null, null)); + assertNotNull(applications.get(appAttemptId0)); + + // Imitating the thread of scheduler that will add and remove apps + final AtomicBoolean finished = new AtomicBoolean(false); + final AtomicBoolean failed = new AtomicBoolean(false); + Thread t = new Thread() { + + @Override + public void run() { + for (int i = 1; i <= size; ++i) { + ApplicationAttemptId appAttemptId + = ApplicationAttemptId.newInstance(appId, i); + try { + applications.put(appAttemptId, ctor.newInstance( + appAttemptId, null, mock(Queue.class), null, null)); + } catch (Exception e) { + failed.set(true); + finished.set(true); + return; + } + } + for (int i = 1; i <= size; ++i) { + ApplicationAttemptId appAttemptId + = ApplicationAttemptId.newInstance(appId, i); + applications.remove(appAttemptId); + } + finished.set(true); + } + }; + t.start(); + + // Imitating the thread of rmappattempt that will get the app + while (!finished.get()) { + assertNotNull(applications.get(appAttemptId0)); + } + assertFalse(failed.get()); + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 17ff7ded850..0b348eaf044 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -72,6 +72,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; @@ -79,6 +80,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemoved import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.resource.Resources; @@ -2196,4 +2198,11 @@ public class TestFairScheduler { assertEquals(1, app.getLiveContainers().size()); } + @Test + public void testConcurrentAccessOnApplications() throws Exception { + FairScheduler fs = new FairScheduler(); + TestCapacityScheduler.verifyConcurrentAccessOnApplications( + fs.applications, FSSchedulerApp.class); + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index b71726a355c..30ce68e73ee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -51,13 +51,15 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; -import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; import org.junit.Before; @@ -414,7 +416,14 @@ public class TestFifoScheduler { LOG.info("--- END: testFifoScheduler ---"); } - + + @Test + public void testConcurrentAccessOnApplications() throws Exception { + FifoScheduler fs = new FifoScheduler(); + TestCapacityScheduler.verifyConcurrentAccessOnApplications( + fs.applications, FiCaSchedulerApp.class); + } + private void checkApplicationResourceUsage(int expected, Application application) { Assert.assertEquals(expected, application.getUsedResources().getMemory()); From 11944e52fdb93e72272e2a9353730fa1603f8352 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Tue, 10 Sep 2013 12:14:49 +0000 Subject: [PATCH 20/21] Amending yarn CHANGES.txt git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1521454 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index b211a18bbb7..d86d998c4f2 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -28,10 +28,6 @@ Release 2.3.0 - UNRELEASED YARN-905. Add state filters to nodes CLI (Wei Yan via Sandy Ryza) - YARN-910. Augmented auxiliary services to listen for container starts and - completions in addition to application events. (Alejandro Abdelnur via - vinodkv) - OPTIMIZATIONS BUG FIXES @@ -96,6 +92,10 @@ Release 2.1.1-beta - UNRELEASED YARN-696. Changed RMWebservice apps call to take in multiple application states. (Trevor Lorimer via vinodkv) + YARN-910. Augmented auxiliary services to listen for container starts and + completions in addition to application events. (Alejandro Abdelnur via + vinodkv) + OPTIMIZATIONS BUG FIXES From bcb865314f8ce89fc03a3e3de487fb6d75ddddd8 Mon Sep 17 00:00:00 2001 From: Bikas Saha Date: Tue, 10 Sep 2013 17:43:51 +0000 Subject: [PATCH 21/21] YARN-1098. Separate out RM services into Always On and Active (Karthik Kambatla via bikas) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1521560 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 2 + .../resourcemanager/ResourceManager.java | 434 +++++++++--------- 2 files changed, 227 insertions(+), 209 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index d86d998c4f2..76365a8b885 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -27,6 +27,8 @@ Release 2.3.0 - UNRELEASED IMPROVEMENTS YARN-905. Add state filters to nodes CLI (Wei Yan via Sandy Ryza) + YARN-1098. Separate out RM services into Always On and Active (Karthik + Kambatla via bikas) OPTIMIZATIONS diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index ee418c1937c..8c0b195f707 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -107,9 +107,18 @@ public class ResourceManager extends CompositeService implements Recoverable { private static final Log LOG = LogFactory.getLog(ResourceManager.class); public static final long clusterTimeStamp = System.currentTimeMillis(); + /** + * "Active" services. Services that need to run only on the Active RM. + * These services are managed (initialized, started, stopped) by the + * {@link CompositeService} RMActiveServices. + * + * RM is active when (1) HA is disabled, or (2) HA is enabled and the RM is + * in Active state. + */ + protected RMActiveServices activeServices; protected ClientToAMTokenSecretManagerInRM clientToAMSecretManager = new ClientToAMTokenSecretManagerInRM(); - + protected RMContainerTokenSecretManager containerTokenSecretManager; protected NMTokenSecretManagerInRM nmTokenSecretManager; @@ -135,6 +144,8 @@ public class ResourceManager extends CompositeService implements Recoverable { protected ResourceTrackerService resourceTracker; private boolean recoveryEnabled; + /** End of Active services */ + private Configuration conf; public ResourceManager() { @@ -147,137 +158,11 @@ public class ResourceManager extends CompositeService implements Recoverable { @Override protected void serviceInit(Configuration conf) throws Exception { - validateConfigs(conf); - this.conf = conf; - this.conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true); - - this.rmDispatcher = createDispatcher(); - addIfService(this.rmDispatcher); - - this.amRmTokenSecretManager = createAMRMTokenSecretManager(conf); - - this.containerAllocationExpirer = new ContainerAllocationExpirer( - this.rmDispatcher); - addService(this.containerAllocationExpirer); - - AMLivelinessMonitor amLivelinessMonitor = createAMLivelinessMonitor(); - addService(amLivelinessMonitor); - - AMLivelinessMonitor amFinishingMonitor = createAMLivelinessMonitor(); - addService(amFinishingMonitor); - - this.containerTokenSecretManager = createContainerTokenSecretManager(conf); - this.nmTokenSecretManager = createNMTokenSecretManager(conf); - - boolean isRecoveryEnabled = conf.getBoolean( - YarnConfiguration.RECOVERY_ENABLED, - YarnConfiguration.DEFAULT_RM_RECOVERY_ENABLED); - - RMStateStore rmStore = null; - if(isRecoveryEnabled) { - recoveryEnabled = true; - rmStore = RMStateStoreFactory.getStore(conf); - } else { - recoveryEnabled = false; - rmStore = new NullRMStateStore(); - } - - try { - rmStore.init(conf); - rmStore.setRMDispatcher(rmDispatcher); - } catch (Exception e) { - // the Exception from stateStore.init() needs to be handled for - // HA and we need to give up master status if we got fenced - LOG.error("Failed to init state store", e); - ExitUtil.terminate(1, e); - } - - if (UserGroupInformation.isSecurityEnabled()) { - this.delegationTokenRenewer = createDelegationTokenRenewer(); - } - - this.rmContext = - new RMContextImpl(this.rmDispatcher, rmStore, - this.containerAllocationExpirer, amLivelinessMonitor, - amFinishingMonitor, delegationTokenRenewer, this.amRmTokenSecretManager, - this.containerTokenSecretManager, this.nmTokenSecretManager, - this.clientToAMSecretManager); - - // Register event handler for NodesListManager - this.nodesListManager = new NodesListManager(this.rmContext); - this.rmDispatcher.register(NodesListManagerEventType.class, - this.nodesListManager); - addService(nodesListManager); - - // Initialize the scheduler - this.scheduler = createScheduler(); - this.schedulerDispatcher = createSchedulerEventDispatcher(); - addIfService(this.schedulerDispatcher); - this.rmDispatcher.register(SchedulerEventType.class, - this.schedulerDispatcher); - - // Register event handler for RmAppEvents - this.rmDispatcher.register(RMAppEventType.class, - new ApplicationEventDispatcher(this.rmContext)); - - // Register event handler for RmAppAttemptEvents - this.rmDispatcher.register(RMAppAttemptEventType.class, - new ApplicationAttemptEventDispatcher(this.rmContext)); - - // Register event handler for RmNodes - this.rmDispatcher.register(RMNodeEventType.class, - new NodeEventDispatcher(this.rmContext)); - - this.nmLivelinessMonitor = createNMLivelinessMonitor(); - addService(this.nmLivelinessMonitor); - - this.resourceTracker = createResourceTrackerService(); - addService(resourceTracker); - - DefaultMetricsSystem.initialize("ResourceManager"); - JvmMetrics.initSingleton("ResourceManager", null); - - try { - this.scheduler.reinitialize(conf, this.rmContext); - } catch (IOException ioe) { - throw new RuntimeException("Failed to initialize scheduler", ioe); - } - - // creating monitors that handle preemption - createPolicyMonitors(); - - masterService = createApplicationMasterService(); - addService(masterService) ; - - this.applicationACLsManager = new ApplicationACLsManager(conf); - - this.rmAppManager = createRMAppManager(); - // Register event handler for RMAppManagerEvents - this.rmDispatcher.register(RMAppManagerEventType.class, - this.rmAppManager); - this.rmDTSecretManager = createRMDelegationTokenSecretManager(this.rmContext); - rmContext.setRMDelegationTokenSecretManager(this.rmDTSecretManager); - clientRM = createClientRMService(); - rmContext.setClientRMService(clientRM); - addService(clientRM); - - adminService = createAdminService(clientRM, masterService, resourceTracker); - addService(adminService); - - this.applicationMasterLauncher = createAMLauncher(); - this.rmDispatcher.register(AMLauncherEventType.class, - this.applicationMasterLauncher); - - addService(applicationMasterLauncher); - if (UserGroupInformation.isSecurityEnabled()) { - addService(delegationTokenRenewer); - delegationTokenRenewer.setRMContext(rmContext); - } - new RMNMInfo(this.rmContext, this.scheduler); - + activeServices = new RMActiveServices(); + addService(activeServices); super.serviceInit(conf); } @@ -378,6 +263,217 @@ public class ResourceManager extends CompositeService implements Recoverable { } } + /** + * RMActiveServices handles all the Active services in the RM. + */ + @Private + class RMActiveServices extends CompositeService { + RMActiveServices() { + super("RMActiveServices"); + } + + @Override + protected void serviceInit(Configuration configuration) throws Exception { + conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true); + + rmDispatcher = createDispatcher(); + addIfService(rmDispatcher); + + amRmTokenSecretManager = createAMRMTokenSecretManager(conf); + + containerAllocationExpirer = new ContainerAllocationExpirer(rmDispatcher); + addService(containerAllocationExpirer); + + AMLivelinessMonitor amLivelinessMonitor = createAMLivelinessMonitor(); + addService(amLivelinessMonitor); + + AMLivelinessMonitor amFinishingMonitor = createAMLivelinessMonitor(); + addService(amFinishingMonitor); + + containerTokenSecretManager = createContainerTokenSecretManager(conf); + nmTokenSecretManager = createNMTokenSecretManager(conf); + + boolean isRecoveryEnabled = conf.getBoolean( + YarnConfiguration.RECOVERY_ENABLED, + YarnConfiguration.DEFAULT_RM_RECOVERY_ENABLED); + + RMStateStore rmStore = null; + if(isRecoveryEnabled) { + recoveryEnabled = true; + rmStore = RMStateStoreFactory.getStore(conf); + } else { + recoveryEnabled = false; + rmStore = new NullRMStateStore(); + } + + try { + rmStore.init(conf); + rmStore.setRMDispatcher(rmDispatcher); + } catch (Exception e) { + // the Exception from stateStore.init() needs to be handled for + // HA and we need to give up master status if we got fenced + LOG.error("Failed to init state store", e); + ExitUtil.terminate(1, e); + } + + if (UserGroupInformation.isSecurityEnabled()) { + delegationTokenRenewer = createDelegationTokenRenewer(); + } + + rmContext = new RMContextImpl( + rmDispatcher, rmStore, containerAllocationExpirer, amLivelinessMonitor, + amFinishingMonitor, delegationTokenRenewer, amRmTokenSecretManager, + containerTokenSecretManager, nmTokenSecretManager, + clientToAMSecretManager); + + // Register event handler for NodesListManager + nodesListManager = new NodesListManager(rmContext); + rmDispatcher.register(NodesListManagerEventType.class, nodesListManager); + addService(nodesListManager); + + // Initialize the scheduler + scheduler = createScheduler(); + schedulerDispatcher = createSchedulerEventDispatcher(); + addIfService(schedulerDispatcher); + rmDispatcher.register(SchedulerEventType.class, schedulerDispatcher); + + // Register event handler for RmAppEvents + rmDispatcher.register(RMAppEventType.class, + new ApplicationEventDispatcher(rmContext)); + + // Register event handler for RmAppAttemptEvents + rmDispatcher.register(RMAppAttemptEventType.class, + new ApplicationAttemptEventDispatcher(rmContext)); + + // Register event handler for RmNodes + rmDispatcher.register( + RMNodeEventType.class, new NodeEventDispatcher(rmContext)); + + nmLivelinessMonitor = createNMLivelinessMonitor(); + addService(nmLivelinessMonitor); + + resourceTracker = createResourceTrackerService(); + addService(resourceTracker); + + DefaultMetricsSystem.initialize("ResourceManager"); + JvmMetrics.initSingleton("ResourceManager", null); + + try { + scheduler.reinitialize(conf, rmContext); + } catch (IOException ioe) { + throw new RuntimeException("Failed to initialize scheduler", ioe); + } + + // creating monitors that handle preemption + createPolicyMonitors(); + + masterService = createApplicationMasterService(); + addService(masterService) ; + + applicationACLsManager = new ApplicationACLsManager(conf); + + rmAppManager = createRMAppManager(); + // Register event handler for RMAppManagerEvents + rmDispatcher.register(RMAppManagerEventType.class, rmAppManager); + rmDTSecretManager = createRMDelegationTokenSecretManager(rmContext); + rmContext.setRMDelegationTokenSecretManager(rmDTSecretManager); + clientRM = createClientRMService(); + rmContext.setClientRMService(clientRM); + addService(clientRM); + + adminService = createAdminService(clientRM, masterService, resourceTracker); + addService(adminService); + + applicationMasterLauncher = createAMLauncher(); + rmDispatcher.register(AMLauncherEventType.class, + applicationMasterLauncher); + + addService(applicationMasterLauncher); + if (UserGroupInformation.isSecurityEnabled()) { + addService(delegationTokenRenewer); + delegationTokenRenewer.setRMContext(rmContext); + } + + new RMNMInfo(rmContext, scheduler); + + super.serviceInit(conf); + } + + @Override + protected void serviceStart() throws Exception { + amRmTokenSecretManager.start(); + containerTokenSecretManager.start(); + nmTokenSecretManager.start(); + + RMStateStore rmStore = rmContext.getStateStore(); + // The state store needs to start irrespective of recoveryEnabled as apps + // need events to move to further states. + rmStore.start(); + + if(recoveryEnabled) { + try { + RMState state = rmStore.loadState(); + recover(state); + } catch (Exception e) { + // the Exception from loadState() needs to be handled for + // HA and we need to give up master status if we got fenced + LOG.error("Failed to load/recover state", e); + ExitUtil.terminate(1, e); + } + } + + startWepApp(); + try { + rmDTSecretManager.startThreads(); + } catch(IOException ie) { + throw new YarnRuntimeException("Failed to start secret manager threads", ie); + } + + if (getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { + String hostname = getConfig().get(YarnConfiguration.RM_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS); + hostname = (hostname.contains(":")) ? hostname.substring(0, hostname.indexOf(":")) : hostname; + int port = webApp.port(); + String resolvedAddress = hostname + ":" + port; + conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, resolvedAddress); + } + + super.serviceStart(); + } + + @Override + protected void serviceStop() throws Exception { + if (webApp != null) { + webApp.stop(); + } + if (rmDTSecretManager != null) { + rmDTSecretManager.stopThreads(); + } + + if (amRmTokenSecretManager != null) { + amRmTokenSecretManager.stop(); + } + if (containerTokenSecretManager != null) { + containerTokenSecretManager.stop(); + } + if(nmTokenSecretManager != null) { + nmTokenSecretManager.stop(); + } + + DefaultMetricsSystem.shutdown(); + + if (rmContext != null) { + RMStateStore store = rmContext.getStateStore(); + try { + store.close(); + } catch (Exception e) { + LOG.error("Error closing store.", e); + } + } + super.serviceStop(); + } + } + @Private public static class SchedulerEventDispatcher extends AbstractService implements EventHandler { @@ -620,54 +716,7 @@ public class ResourceManager extends CompositeService implements Recoverable { throw new YarnRuntimeException("Failed to login", ie); } - this.amRmTokenSecretManager.start(); - this.containerTokenSecretManager.start(); - this.nmTokenSecretManager.start(); - - RMStateStore rmStore = rmContext.getStateStore(); - // The state store needs to start irrespective of recoveryEnabled as apps - // need events to move to further states. - rmStore.start(); - - if(recoveryEnabled) { - try { - RMState state = rmStore.loadState(); - recover(state); - } catch (Exception e) { - // the Exception from loadState() needs to be handled for - // HA and we need to give up master status if we got fenced - LOG.error("Failed to load/recover state", e); - ExitUtil.terminate(1, e); - } - } - - startWepApp(); - try { - rmDTSecretManager.startThreads(); - } catch(IOException ie) { - throw new YarnRuntimeException("Failed to start secret manager threads", ie); - } - - if (getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { - String hostname = getConfig().get(YarnConfiguration.RM_WEBAPP_ADDRESS, - YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS); - hostname = (hostname.contains(":")) ? hostname.substring(0, hostname.indexOf(":")) : hostname; - int port = webApp.port(); - String resolvedAddress = hostname + ":" + port; - conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, resolvedAddress); - } - super.serviceStart(); - - /*synchronized(shutdown) { - try { - while(!shutdown.get()) { - shutdown.wait(); - } - } catch(InterruptedException ie) { - LOG.info("Interrupted while waiting", ie); - } - }*/ } protected void doSecureLogin() throws IOException { @@ -677,39 +726,6 @@ public class ResourceManager extends CompositeService implements Recoverable { @Override protected void serviceStop() throws Exception { - if (webApp != null) { - webApp.stop(); - } - if (rmDTSecretManager != null) { - rmDTSecretManager.stopThreads(); - } - - if (amRmTokenSecretManager != null) { - this.amRmTokenSecretManager.stop(); - } - if (containerTokenSecretManager != null) { - this.containerTokenSecretManager.stop(); - } - if(nmTokenSecretManager != null) { - nmTokenSecretManager.stop(); - } - - /*synchronized(shutdown) { - shutdown.set(true); - shutdown.notifyAll(); - }*/ - - DefaultMetricsSystem.shutdown(); - - if (rmContext != null) { - RMStateStore store = rmContext.getStateStore(); - try { - store.close(); - } catch (Exception e) { - LOG.error("Error closing store.", e); - } - } - super.serviceStop(); }