diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 7a7106197d9..4e6dc469ebe 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -312,6 +312,11 @@ Release 2.4.0 - UNRELEASED HADOOP-10295. Allow distcp to automatically identify the checksum type of source files and use it for the target. (jing9 and Laurent Goujon) + HADOOP-10333. Fix grammatical error in overview.html document. + (René Nyffenegger via suresh) + + HADOOP-10343. Change info to debug log in LossyRetryInvocationHandler. (arpit) + OPTIMIZATIONS BUG FIXES @@ -328,15 +333,36 @@ Release 2.4.0 - UNRELEASED HADOOP-10330. TestFrameDecoder fails if it cannot bind port 12345. (Arpit Agarwal) -Release 2.3.0 - UNRELEASED + HADOOP-10326. M/R jobs can not access S3 if Kerberos is enabled. (bc Wong + via atm) + + HADOOP-10338. Cannot get the FileStatus of the root inode from the new + Globber (cmccabe) + + HADOOP-10249. LdapGroupsMapping should trim ldap password read from file. + (Dilli Armugam via suresh) + +Release 2.3.1 - UNRELEASED INCOMPATIBLE CHANGES - HADOOP-8545. Filesystem Implementation for OpenStack Swift - (Dmitry Mezhensky, David Dobbins, Stevel via stevel) + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + +Release 2.3.0 - 2014-02-18 + + INCOMPATIBLE CHANGES NEW FEATURES + HADOOP-8545. Filesystem Implementation for OpenStack Swift + (Dmitry Mezhensky, David Dobbins, Stevel via stevel) + IMPROVEMENTS HADOOP-10046. Print a log message when SSL is enabled. diff --git a/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html b/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html index efbaeae4b14..d2b6156573d 100644 --- a/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html +++ b/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html @@ -1,3 +1,2953 @@ + +Hadoop 2.3.0 Release Notes + + + +

Hadoop 2.3.0 Release Notes

+These release notes include new developer and user-facing incompatibilities, features, and major improvements. + +

Changes since Hadoop 2.2.0

+ + - diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js index 946d21815da..39450043cda 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js @@ -28,7 +28,7 @@ {"name": "nn", "url": "/jmx?qry=Hadoop:service=NameNode,name=NameNodeInfo"}, {"name": "nnstat", "url": "/jmx?qry=Hadoop:service=NameNode,name=NameNodeStatus"}, {"name": "fs", "url": "/jmx?qry=Hadoop:service=NameNode,name=FSNamesystemState"}, - {"name": "mem", "url": "/jmx?qry=java.lang:type=Memory"}, + {"name": "mem", "url": "/jmx?qry=java.lang:type=Memory"} ]; var HELPERS = { @@ -166,14 +166,29 @@ $('#ui-tabs a[href="#tab-snapshot"]').click(load_snapshot_info); - var hash = window.location.hash; - if (hash === "#tab-datanode") { - load_datanode_info(); - } else if (hash === "#tab-snapshot") { - load_snapshot_info(); - } else if (hash === "#tab-startup-progress") { - load_startup_progress(); - } else { - load_overview(); + function load_page() { + var hash = window.location.hash; + switch(hash) { + case "#tab-datanode": + load_datanode_info(); + break; + case "#tab-snapshot": + load_snapshot_info(); + break; + case "#tab-startup-progress": + load_startup_progress(); + break; + case "#tab-overview": + load_overview(); + break; + default: + window.location.hash = "tab-overview"; + break; + } } + load_page(); + + $(window).bind('hashchange', function () { + load_page(); + }); })(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html index 50c7dfe46b4..f9c339748f9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html @@ -1,3 +1,5 @@ + - diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js index 2e1af80c70e..9d1ca663527 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js @@ -35,8 +35,8 @@ } if (sticky) { - var exec = ((parms.perm % 10) & 1) == 1; - res[res.length - 1] = exec ? 't' : 'T'; + var otherExec = ((ctx.current().permission % 10) & 1) == 1; + res = res.substr(0, res.length - 1) + (otherExec ? 't' : 'T'); } chunk.write(dir + res); @@ -52,6 +52,18 @@ $('#alert-panel').show(); } + $(window).bind('hashchange', function () { + $('#alert-panel').hide(); + + var dir = window.location.hash.slice(1); + if(dir == "") { + dir = "/"; + } + if(current_directory != dir) { + browse_directory(dir); + } + }); + function network_error_handler(url) { return function (jqxhr, text, err) { var msg = '

Failed to retreive data from ' + url + ', cause: ' + err + '

'; @@ -145,6 +157,7 @@ current_directory = dir; $('#directory').val(dir); + window.location.hash = dir; dust.render('explorer', base.push(d), function(err, out) { $('#panel').html(out); @@ -169,7 +182,12 @@ var b = function() { browse_directory($('#directory').val()); }; $('#btn-nav-directory').click(b); - browse_directory('/'); + var dir = window.location.hash.slice(1); + if(dir == "") { + window.location.hash = "/"; + } else { + browse_directory(dir); + } } init(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java index c4045c35fb7..6f0fafa2628 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java @@ -28,32 +28,39 @@ import java.util.EnumSet; import java.util.Random; import org.apache.commons.lang.SystemUtils; +import org.apache.commons.lang.mutable.MutableBoolean; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.ExtendedBlockId; +import org.apache.hadoop.hdfs.ClientContext; +import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.client.ClientMmap; -import org.apache.hadoop.hdfs.client.ClientMmapManager; import org.apache.hadoop.hdfs.client.HdfsDataInputStream; +import org.apache.hadoop.hdfs.client.ShortCircuitCache; +import org.apache.hadoop.hdfs.client.ShortCircuitCache.CacheVisitor; +import org.apache.hadoop.hdfs.client.ShortCircuitReplica; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.io.ByteBufferPool; -import org.apache.hadoop.io.ElasticByteBufferPool; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.net.unix.DomainSocket; import org.apache.hadoop.net.unix.TemporarySocketDirectory; +import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; import org.junit.Assume; import org.junit.BeforeClass; import org.junit.Test; +import java.util.Map; + import com.google.common.base.Preconditions; import com.google.common.base.Supplier; @@ -250,17 +257,39 @@ public class TestEnhancedByteBufferAccess { } } - private static class CountingVisitor - implements ClientMmapManager.ClientMmapVisitor { - int count = 0; + private static class CountingVisitor implements CacheVisitor { + private final int expectedNumOutstandingMmaps; + private final int expectedNumReplicas; + private final int expectedNumEvictable; + private final int expectedNumMmapedEvictable; - @Override - public void accept(ClientMmap mmap) { - count++; + CountingVisitor(int expectedNumOutstandingMmaps, + int expectedNumReplicas, int expectedNumEvictable, + int expectedNumMmapedEvictable) { + this.expectedNumOutstandingMmaps = expectedNumOutstandingMmaps; + this.expectedNumReplicas = expectedNumReplicas; + this.expectedNumEvictable = expectedNumEvictable; + this.expectedNumMmapedEvictable = expectedNumMmapedEvictable; } - public void reset() { - count = 0; + @Override + public void visit(int numOutstandingMmaps, + Map replicas, + Map failedLoads, + Map evictable, + Map evictableMmapped) { + if (expectedNumOutstandingMmaps >= 0) { + Assert.assertEquals(expectedNumOutstandingMmaps, numOutstandingMmaps); + } + if (expectedNumReplicas >= 0) { + Assert.assertEquals(expectedNumReplicas, replicas.size()); + } + if (expectedNumEvictable >= 0) { + Assert.assertEquals(expectedNumEvictable, evictable.size()); + } + if (expectedNumMmapedEvictable >= 0) { + Assert.assertEquals(expectedNumMmapedEvictable, evictableMmapped.size()); + } } } @@ -271,105 +300,98 @@ public class TestEnhancedByteBufferAccess { final Path TEST_PATH = new Path("/a"); final int TEST_FILE_LENGTH = 16385; final int RANDOM_SEED = 23453; + final String CONTEXT = "testZeroCopyMmapCacheContext"; FSDataInputStream fsIn = null; - ByteBuffer results[] = { null, null, null, null, null }; - + ByteBuffer results[] = { null, null, null, null }; + DistributedFileSystem fs = null; + conf.set(DFSConfigKeys.DFS_CLIENT_CONTEXT, CONTEXT); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + cluster.waitActive(); + fs = cluster.getFileSystem(); + DFSTestUtil.createFile(fs, TEST_PATH, + TEST_FILE_LENGTH, (short)1, RANDOM_SEED); try { - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); - cluster.waitActive(); - fs = cluster.getFileSystem(); - DFSTestUtil.createFile(fs, TEST_PATH, - TEST_FILE_LENGTH, (short)1, RANDOM_SEED); - try { - DFSTestUtil.waitReplication(fs, TEST_PATH, (short)1); - } catch (InterruptedException e) { - Assert.fail("unexpected InterruptedException during " + - "waitReplication: " + e); - } catch (TimeoutException e) { - Assert.fail("unexpected TimeoutException during " + - "waitReplication: " + e); - } - fsIn = fs.open(TEST_PATH); - byte original[] = new byte[TEST_FILE_LENGTH]; - IOUtils.readFully(fsIn, original, 0, TEST_FILE_LENGTH); - fsIn.close(); - fsIn = fs.open(TEST_PATH); - final ClientMmapManager mmapManager = fs.getClient().getMmapManager(); - final CountingVisitor countingVisitor = new CountingVisitor(); - mmapManager.visitMmaps(countingVisitor); - Assert.assertEquals(0, countingVisitor.count); - mmapManager.visitEvictable(countingVisitor); - Assert.assertEquals(0, countingVisitor.count); - results[0] = fsIn.read(null, 4096, - EnumSet.of(ReadOption.SKIP_CHECKSUMS)); - fsIn.seek(0); - results[1] = fsIn.read(null, 4096, - EnumSet.of(ReadOption.SKIP_CHECKSUMS)); - mmapManager.visitMmaps(countingVisitor); - Assert.assertEquals(1, countingVisitor.count); - countingVisitor.reset(); - mmapManager.visitEvictable(countingVisitor); - Assert.assertEquals(0, countingVisitor.count); - countingVisitor.reset(); - - // The mmaps should be of the first block of the file. - final ExtendedBlock firstBlock = DFSTestUtil.getFirstBlock(fs, TEST_PATH); - mmapManager.visitMmaps(new ClientMmapManager.ClientMmapVisitor() { - @Override - public void accept(ClientMmap mmap) { - Assert.assertEquals(firstBlock, mmap.getBlock()); - } - }); - - // Read more blocks. - results[2] = fsIn.read(null, 4096, - EnumSet.of(ReadOption.SKIP_CHECKSUMS)); - results[3] = fsIn.read(null, 4096, - EnumSet.of(ReadOption.SKIP_CHECKSUMS)); - try { - results[4] = fsIn.read(null, 4096, - EnumSet.of(ReadOption.SKIP_CHECKSUMS)); - Assert.fail("expected UnsupportedOperationException"); - } catch (UnsupportedOperationException e) { - // expected - } - - // we should have 3 mmaps, 0 evictable - mmapManager.visitMmaps(countingVisitor); - Assert.assertEquals(3, countingVisitor.count); - countingVisitor.reset(); - mmapManager.visitEvictable(countingVisitor); - Assert.assertEquals(0, countingVisitor.count); - - // After we close the cursors, the mmaps should be evictable for - // a brief period of time. Then, they should be closed (we're - // using a very quick timeout) - for (ByteBuffer buffer : results) { - if (buffer != null) { - fsIn.releaseBuffer(buffer); - } - } - GenericTestUtils.waitFor(new Supplier() { - public Boolean get() { - countingVisitor.reset(); - try { - mmapManager.visitEvictable(countingVisitor); - } catch (InterruptedException e) { - e.printStackTrace(); - return false; - } - return (0 == countingVisitor.count); - } - }, 10, 10000); - countingVisitor.reset(); - mmapManager.visitMmaps(countingVisitor); - Assert.assertEquals(0, countingVisitor.count); - } finally { - if (fsIn != null) fsIn.close(); - if (fs != null) fs.close(); - if (cluster != null) cluster.shutdown(); + DFSTestUtil.waitReplication(fs, TEST_PATH, (short)1); + } catch (InterruptedException e) { + Assert.fail("unexpected InterruptedException during " + + "waitReplication: " + e); + } catch (TimeoutException e) { + Assert.fail("unexpected TimeoutException during " + + "waitReplication: " + e); } + fsIn = fs.open(TEST_PATH); + byte original[] = new byte[TEST_FILE_LENGTH]; + IOUtils.readFully(fsIn, original, 0, TEST_FILE_LENGTH); + fsIn.close(); + fsIn = fs.open(TEST_PATH); + final ShortCircuitCache cache = ClientContext.get( + CONTEXT, new DFSClient.Conf(conf)). getShortCircuitCache(); + cache.accept(new CountingVisitor(0, 5, 5, 0)); + results[0] = fsIn.read(null, 4096, + EnumSet.of(ReadOption.SKIP_CHECKSUMS)); + fsIn.seek(0); + results[1] = fsIn.read(null, 4096, + EnumSet.of(ReadOption.SKIP_CHECKSUMS)); + + // The mmap should be of the first block of the file. + final ExtendedBlock firstBlock = + DFSTestUtil.getFirstBlock(fs, TEST_PATH); + cache.accept(new CacheVisitor() { + @Override + public void visit(int numOutstandingMmaps, + Map replicas, + Map failedLoads, + Map evictable, + Map evictableMmapped) { + ShortCircuitReplica replica = replicas.get( + new ExtendedBlockId(firstBlock.getBlockId(), firstBlock.getBlockPoolId())); + Assert.assertNotNull(replica); + Assert.assertTrue(replica.hasMmap()); + // The replica should not yet be evictable, since we have it open. + Assert.assertNull(replica.getEvictableTimeNs()); + } + }); + + // Read more blocks. + results[2] = fsIn.read(null, 4096, + EnumSet.of(ReadOption.SKIP_CHECKSUMS)); + results[3] = fsIn.read(null, 4096, + EnumSet.of(ReadOption.SKIP_CHECKSUMS)); + + // we should have 3 mmaps, 1 evictable + cache.accept(new CountingVisitor(3, 5, 2, 0)); + + // After we close the cursors, the mmaps should be evictable for + // a brief period of time. Then, they should be closed (we're + // using a very quick timeout) + for (ByteBuffer buffer : results) { + if (buffer != null) { + fsIn.releaseBuffer(buffer); + } + } + fsIn.close(); + GenericTestUtils.waitFor(new Supplier() { + public Boolean get() { + final MutableBoolean finished = new MutableBoolean(false); + cache.accept(new CacheVisitor() { + @Override + public void visit(int numOutstandingMmaps, + Map replicas, + Map failedLoads, + Map evictable, + Map evictableMmapped) { + finished.setValue(evictableMmapped.isEmpty()); + } + }); + return finished.booleanValue(); + } + }, 10, 60000); + + cache.accept(new CountingVisitor(0, -1, -1, -1)); + + fs.close(); + cluster.shutdown(); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java index 8eb9847ebb5..5ba797c8b01 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java @@ -21,6 +21,7 @@ import static org.junit.Assert.*; import java.io.IOException; import java.security.PrivilegedExceptionAction; +import java.util.UUID; import java.util.regex.Pattern; import org.apache.commons.lang.StringUtils; @@ -1175,4 +1176,32 @@ public class TestGlobPaths { public void testReservedHdfsPathsOnFC() throws Exception { testOnFileContext(new TestReservedHdfsPaths()); } + + /** + * Test trying to glob the root. Regression test for HDFS-5888. + **/ + private static class TestGlobRoot implements FSTestWrapperGlobTest { + public void run(FSTestWrapper wrap, FSTestWrapper unprivilegedWrap, + FileSystem fs, FileContext fc) throws Exception { + final Path rootPath = new Path("/"); + FileStatus oldRootStatus = wrap.getFileStatus(rootPath); + String newOwner = UUID.randomUUID().toString(); + wrap.setOwner(new Path("/"), newOwner, null); + FileStatus[] status = + wrap.globStatus(rootPath, new AcceptAllPathFilter()); + Assert.assertEquals(1, status.length); + Assert.assertEquals(newOwner, status[0].getOwner()); + wrap.setOwner(new Path("/"), oldRootStatus.getOwner(), null); + } + } + + @Test + public void testGlobRootOnFS() throws Exception { + testOnFileSystem(new TestGlobRoot()); + } + + @Test + public void testGlobRootOnFC() throws Exception { + testOnFileContext(new TestGlobRoot()); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/BlockReaderTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/BlockReaderTestUtil.java index ac17915df5f..0e511cd5f71 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/BlockReaderTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/BlockReaderTestUtil.java @@ -28,8 +28,12 @@ import java.net.Socket; import java.util.List; import java.util.Random; +import org.apache.commons.io.IOUtils; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.client.ShortCircuitCache; +import org.apache.hadoop.hdfs.client.ShortCircuitReplica; +import org.apache.hadoop.hdfs.net.Peer; import org.apache.hadoop.hdfs.net.TcpPeerServer; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; @@ -38,6 +42,8 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.datanode.CachingStrategy; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.net.NetUtils; +import org.apache.log4j.Level; +import org.apache.log4j.LogManager; /** * A helper class to setup the cluster, and get to BlockReader and DataNode for a block. @@ -141,22 +147,54 @@ public class BlockReaderTestUtil { */ public BlockReader getBlockReader(LocatedBlock testBlock, int offset, int lenToRead) throws IOException { + return getBlockReader(cluster, testBlock, offset, lenToRead); + } + + /** + * Get a BlockReader for the given block. + */ + public static BlockReader getBlockReader(MiniDFSCluster cluster, + LocatedBlock testBlock, int offset, int lenToRead) throws IOException { InetSocketAddress targetAddr = null; - Socket sock = null; ExtendedBlock block = testBlock.getBlock(); DatanodeInfo[] nodes = testBlock.getLocations(); targetAddr = NetUtils.createSocketAddr(nodes[0].getXferAddr()); - sock = NetUtils.getDefaultSocketFactory(conf).createSocket(); - sock.connect(targetAddr, HdfsServerConstants.READ_TIMEOUT); - sock.setSoTimeout(HdfsServerConstants.READ_TIMEOUT); - return BlockReaderFactory.newBlockReader( - new DFSClient.Conf(conf), - targetAddr.toString()+ ":" + block.getBlockId(), block, - testBlock.getBlockToken(), - offset, lenToRead, - true, "BlockReaderTestUtil", TcpPeerServer.peerFromSocket(sock), - nodes[0], null, null, null, false, CachingStrategy.newDefaultStrategy()); + final DistributedFileSystem fs = cluster.getFileSystem(); + return new BlockReaderFactory(fs.getClient().getConf()). + setInetSocketAddress(targetAddr). + setBlock(block). + setFileName(targetAddr.toString()+ ":" + block.getBlockId()). + setBlockToken(testBlock.getBlockToken()). + setStartOffset(offset). + setLength(lenToRead). + setVerifyChecksum(true). + setClientName("BlockReaderTestUtil"). + setDatanodeInfo(nodes[0]). + setClientCacheContext(ClientContext.getFromConf(fs.getConf())). + setCachingStrategy(CachingStrategy.newDefaultStrategy()). + setConfiguration(fs.getConf()). + setAllowShortCircuitLocalReads(true). + setRemotePeerFactory(new RemotePeerFactory() { + @Override + public Peer newConnectedPeer(InetSocketAddress addr) + throws IOException { + Peer peer = null; + Socket sock = NetUtils. + getDefaultSocketFactory(fs.getConf()).createSocket(); + try { + sock.connect(addr, HdfsServerConstants.READ_TIMEOUT); + sock.setSoTimeout(HdfsServerConstants.READ_TIMEOUT); + peer = TcpPeerServer.peerFromSocket(sock); + } finally { + if (peer == null) { + IOUtils.closeQuietly(sock); + } + } + return peer; + } + }). + build(); } /** @@ -167,4 +205,13 @@ public class BlockReaderTestUtil { int ipcport = nodes[0].getIpcPort(); return cluster.getDataNode(ipcport); } -} + + public static void enableBlockReaderFactoryTracing() { + LogManager.getLogger(BlockReaderFactory.class.getName()).setLevel( + Level.TRACE); + LogManager.getLogger(ShortCircuitCache.class.getName()).setLevel( + Level.TRACE); + LogManager.getLogger(ShortCircuitReplica.class.getName()).setLevel( + Level.TRACE); + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index 09bf8497d08..09ea0499e74 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -186,10 +186,26 @@ public class DFSTestUtil { } } - public static String readFile(FileSystem fs, Path fileName) throws IOException { + public static String readFile(FileSystem fs, Path fileName) + throws IOException { + byte buf[] = readFileBuffer(fs, fileName); + return new String(buf, 0, buf.length); + } + + public static byte[] readFileBuffer(FileSystem fs, Path fileName) + throws IOException { ByteArrayOutputStream os = new ByteArrayOutputStream(); - IOUtils.copyBytes(fs.open(fileName), os, 1024, true); - return os.toString(); + try { + FSDataInputStream in = fs.open(fileName); + try { + IOUtils.copyBytes(fs.open(fileName), os, 1024, true); + return os.toByteArray(); + } finally { + in.close(); + } + } finally { + os.close(); + } } public static void createFile(FileSystem fs, Path fileName, long fileLen, @@ -231,6 +247,13 @@ public class DFSTestUtil { } } + public static byte[] calculateFileContentsFromSeed(long seed, int length) { + Random rb = new Random(seed); + byte val[] = new byte[length]; + rb.nextBytes(val); + return val; + } + /** check if the files have been copied correctly. */ public boolean checkFiles(FileSystem fs, String topdir) throws IOException { Path root = new Path(topdir); @@ -550,8 +573,12 @@ public class DFSTestUtil { public static ExtendedBlock getFirstBlock(FileSystem fs, Path path) throws IOException { HdfsDataInputStream in = (HdfsDataInputStream) fs.open(path); - in.readByte(); - return in.getCurrentBlock(); + try { + in.readByte(); + return in.getCurrentBlock(); + } finally { + in.close(); + } } public static List getAllBlocks(FSDataInputStream in) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderFactory.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderFactory.java new file mode 100644 index 00000000000..6b496e21865 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderFactory.java @@ -0,0 +1,285 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import java.io.File; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.logging.LogFactory; +import org.apache.commons.logging.Log; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.client.ShortCircuitCache; +import org.apache.hadoop.hdfs.client.ShortCircuitReplicaInfo; +import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import org.apache.hadoop.net.unix.DomainSocket; +import org.apache.hadoop.net.unix.TemporarySocketDirectory; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.google.common.util.concurrent.Uninterruptibles; + +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CONTEXT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_DOMAIN_SOCKET_DATA_TRAFFIC; + +public class TestBlockReaderFactory { + static final Log LOG = LogFactory.getLog(TestBlockReaderFactory.class); + + @Before + public void init() { + DomainSocket.disableBindPathValidation(); + } + + @After + public void cleanup() { + DFSInputStream.tcpReadsDisabledForTesting = false; + BlockReaderFactory.createShortCircuitReplicaInfoCallback = null; + } + + private static Configuration createShortCircuitConf(String testName, + TemporarySocketDirectory sockDir) { + Configuration conf = new Configuration(); + conf.set(DFS_CLIENT_CONTEXT, testName); + conf.setLong(DFS_BLOCK_SIZE_KEY, 4096); + conf.set(DFS_DOMAIN_SOCKET_PATH_KEY, new File(sockDir.getDir(), + testName + "._PORT").getAbsolutePath()); + conf.setBoolean(DFS_CLIENT_READ_SHORTCIRCUIT_KEY, true); + conf.setBoolean(DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_KEY, + false); + conf.setBoolean(DFS_CLIENT_DOMAIN_SOCKET_DATA_TRAFFIC, false); + return conf; + } + + /** + * If we have a UNIX domain socket configured, + * and we have dfs.client.domain.socket.data.traffic set to true, + * and short-circuit access fails, we should still be able to pass + * data traffic over the UNIX domain socket. Test this. + */ + @Test(timeout=60000) + public void testFallbackFromShortCircuitToUnixDomainTraffic() + throws Exception { + DFSInputStream.tcpReadsDisabledForTesting = true; + TemporarySocketDirectory sockDir = new TemporarySocketDirectory(); + + // The server is NOT configured with short-circuit local reads; + // the client is. Both support UNIX domain reads. + Configuration clientConf = createShortCircuitConf( + "testFallbackFromShortCircuitToUnixDomainTraffic", sockDir); + clientConf.setBoolean(DFS_CLIENT_DOMAIN_SOCKET_DATA_TRAFFIC, true); + Configuration serverConf = new Configuration(clientConf); + serverConf.setBoolean(DFS_CLIENT_READ_SHORTCIRCUIT_KEY, false); + + MiniDFSCluster cluster = + new MiniDFSCluster.Builder(serverConf).numDataNodes(1).build(); + cluster.waitActive(); + FileSystem dfs = FileSystem.get(cluster.getURI(0), clientConf); + String TEST_FILE = "/test_file"; + final int TEST_FILE_LEN = 8193; + final int SEED = 0xFADED; + DFSTestUtil.createFile(dfs, new Path(TEST_FILE), TEST_FILE_LEN, + (short)1, SEED); + byte contents[] = DFSTestUtil.readFileBuffer(dfs, new Path(TEST_FILE)); + byte expected[] = DFSTestUtil. + calculateFileContentsFromSeed(SEED, TEST_FILE_LEN); + Assert.assertTrue(Arrays.equals(contents, expected)); + cluster.shutdown(); + sockDir.close(); + } + + /** + * Test the case where we have multiple threads waiting on the + * ShortCircuitCache delivering a certain ShortCircuitReplica. + * + * In this case, there should only be one call to + * createShortCircuitReplicaInfo. This one replica should be shared + * by all threads. + */ + @Test(timeout=60000) + public void testMultipleWaitersOnShortCircuitCache() + throws Exception { + final CountDownLatch latch = new CountDownLatch(1); + final AtomicBoolean creationIsBlocked = new AtomicBoolean(true); + final AtomicBoolean testFailed = new AtomicBoolean(false); + DFSInputStream.tcpReadsDisabledForTesting = true; + BlockReaderFactory.createShortCircuitReplicaInfoCallback = + new ShortCircuitCache.ShortCircuitReplicaCreator() { + @Override + public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() { + Uninterruptibles.awaitUninterruptibly(latch); + if (!creationIsBlocked.compareAndSet(true, false)) { + Assert.fail("there were multiple calls to " + + "createShortCircuitReplicaInfo. Only one was expected."); + } + return null; + } + }; + TemporarySocketDirectory sockDir = new TemporarySocketDirectory(); + Configuration conf = createShortCircuitConf( + "testMultipleWaitersOnShortCircuitCache", sockDir); + MiniDFSCluster cluster = + new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + cluster.waitActive(); + final DistributedFileSystem dfs = cluster.getFileSystem(); + final String TEST_FILE = "/test_file"; + final int TEST_FILE_LEN = 4000; + final int SEED = 0xFADED; + final int NUM_THREADS = 10; + DFSTestUtil.createFile(dfs, new Path(TEST_FILE), TEST_FILE_LEN, + (short)1, SEED); + Runnable readerRunnable = new Runnable() { + @Override + public void run() { + try { + byte contents[] = DFSTestUtil.readFileBuffer(dfs, new Path(TEST_FILE)); + Assert.assertFalse(creationIsBlocked.get()); + byte expected[] = DFSTestUtil. + calculateFileContentsFromSeed(SEED, TEST_FILE_LEN); + Assert.assertTrue(Arrays.equals(contents, expected)); + } catch (Throwable e) { + LOG.error("readerRunnable error", e); + testFailed.set(true); + } + } + }; + Thread threads[] = new Thread[NUM_THREADS]; + for (int i = 0; i < NUM_THREADS; i++) { + threads[i] = new Thread(readerRunnable); + threads[i].start(); + } + Thread.sleep(500); + latch.countDown(); + for (int i = 0; i < NUM_THREADS; i++) { + Uninterruptibles.joinUninterruptibly(threads[i]); + } + cluster.shutdown(); + sockDir.close(); + Assert.assertFalse(testFailed.get()); + } + + /** + * Test the case where we have a failure to complete a short circuit read + * that occurs, and then later on, we have a success. + * Any thread waiting on a cache load should receive the failure (if it + * occurs); however, the failure result should not be cached. We want + * to be able to retry later and succeed. + */ + @Test(timeout=60000) + public void testShortCircuitCacheTemporaryFailure() + throws Exception { + BlockReaderTestUtil.enableBlockReaderFactoryTracing(); + final AtomicBoolean replicaCreationShouldFail = new AtomicBoolean(true); + final AtomicBoolean testFailed = new AtomicBoolean(false); + DFSInputStream.tcpReadsDisabledForTesting = true; + BlockReaderFactory.createShortCircuitReplicaInfoCallback = + new ShortCircuitCache.ShortCircuitReplicaCreator() { + @Override + public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() { + if (replicaCreationShouldFail.get()) { + // Insert a short delay to increase the chance that one client + // thread waits for the other client thread's failure via + // a condition variable. + Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); + return new ShortCircuitReplicaInfo(); + } + return null; + } + }; + TemporarySocketDirectory sockDir = new TemporarySocketDirectory(); + Configuration conf = createShortCircuitConf( + "testShortCircuitCacheTemporaryFailure", sockDir); + final MiniDFSCluster cluster = + new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + cluster.waitActive(); + final DistributedFileSystem dfs = cluster.getFileSystem(); + final String TEST_FILE = "/test_file"; + final int TEST_FILE_LEN = 4000; + final int NUM_THREADS = 2; + final int SEED = 0xFADED; + final CountDownLatch gotFailureLatch = new CountDownLatch(NUM_THREADS); + final CountDownLatch shouldRetryLatch = new CountDownLatch(1); + DFSTestUtil.createFile(dfs, new Path(TEST_FILE), TEST_FILE_LEN, + (short)1, SEED); + Runnable readerRunnable = new Runnable() { + @Override + public void run() { + try { + // First time should fail. + List locatedBlocks = + cluster.getNameNode().getRpcServer().getBlockLocations( + TEST_FILE, 0, TEST_FILE_LEN).getLocatedBlocks(); + LocatedBlock lblock = locatedBlocks.get(0); // first block + BlockReader blockReader = null; + try { + blockReader = BlockReaderTestUtil. + getBlockReader(cluster, lblock, 0, TEST_FILE_LEN); + Assert.fail("expected getBlockReader to fail the first time."); + } catch (Throwable t) { + Assert.assertTrue("expected to see 'TCP reads were disabled " + + "for testing' in exception " + t, t.getMessage().contains( + "TCP reads were disabled for testing")); + } finally { + if (blockReader != null) blockReader.close(); // keep findbugs happy + } + gotFailureLatch.countDown(); + shouldRetryLatch.await(); + + // Second time should succeed. + try { + blockReader = BlockReaderTestUtil. + getBlockReader(cluster, lblock, 0, TEST_FILE_LEN); + } catch (Throwable t) { + LOG.error("error trying to retrieve a block reader " + + "the second time.", t); + throw t; + } finally { + if (blockReader != null) blockReader.close(); + } + } catch (Throwable t) { + LOG.error("getBlockReader failure", t); + testFailed.set(true); + } + } + }; + Thread threads[] = new Thread[NUM_THREADS]; + for (int i = 0; i < NUM_THREADS; i++) { + threads[i] = new Thread(readerRunnable); + threads[i].start(); + } + gotFailureLatch.await(); + replicaCreationShouldFail.set(false); + shouldRetryLatch.countDown(); + for (int i = 0; i < NUM_THREADS; i++) { + Uninterruptibles.joinUninterruptibly(threads[i]); + } + cluster.shutdown(); + sockDir.close(); + Assert.assertFalse(testFailed.get()); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocal.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocal.java index bfca9d799b4..03dced7bee0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocal.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocal.java @@ -30,13 +30,15 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.client.HdfsDataInputStream; +import org.apache.hadoop.hdfs.client.ShortCircuitCache; +import org.apache.hadoop.hdfs.client.ShortCircuitReplica; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; -import org.apache.hadoop.hdfs.server.datanode.BlockMetadataHeader; import org.apache.hadoop.hdfs.server.datanode.CachingStrategy; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.net.unix.DomainSocket; import org.apache.hadoop.net.unix.TemporarySocketDirectory; +import org.apache.hadoop.util.Time; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Assume; @@ -155,6 +157,8 @@ public class TestBlockReaderLocal { File metaFile = MiniDFSCluster.getBlockMetadataFile(0, block); DatanodeID datanodeID = cluster.getDataNodes().get(0).getDatanodeId(); + ShortCircuitCache shortCircuitCache = + ClientContext.getFromConf(conf).getShortCircuitCache(); cluster.shutdown(); cluster = null; test.setup(dataFile, checksum); @@ -164,16 +168,17 @@ public class TestBlockReaderLocal { }; dataIn = streams[0]; metaIn = streams[1]; + ExtendedBlockId key = new ExtendedBlockId(block.getBlockId(), block.getBlockPoolId()); + ShortCircuitReplica replica = new ShortCircuitReplica( + key, dataIn, metaIn, shortCircuitCache, Time.now()); blockReaderLocal = new BlockReaderLocal.Builder( new DFSClient.Conf(conf)). setFilename(TEST_PATH.getName()). setBlock(block). - setStreams(streams). + setShortCircuitReplica(replica). setDatanodeID(datanodeID). setCachingStrategy(new CachingStrategy(false, readahead)). setVerifyChecksum(checksum). - setBlockMetadataHeader(BlockMetadataHeader.preadHeader( - metaIn.getChannel())). build(); dataIn = null; metaIn = null; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestConnCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestConnCache.java index cffd91dfa47..c518c8c5aec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestConnCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestConnCache.java @@ -25,18 +25,8 @@ import java.net.InetSocketAddress; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.protocol.DatanodeInfo; -import org.apache.hadoop.hdfs.protocol.ExtendedBlock; -import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; -import org.apache.hadoop.hdfs.server.datanode.CachingStrategy; -import org.apache.hadoop.hdfs.net.Peer; -import org.apache.hadoop.security.token.Token; import org.junit.Assert; import org.junit.Test; -import org.mockito.Matchers; -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; /** * This class tests the client connection caching in a single node @@ -48,30 +38,6 @@ public class TestConnCache { static final int BLOCK_SIZE = 4096; static final int FILE_SIZE = 3 * BLOCK_SIZE; - /** - * A mock Answer to remember the BlockReader used. - * - * It verifies that all invocation to DFSInputStream.getBlockReader() - * use the same peer. - */ - private class MockGetBlockReader implements Answer { - public RemoteBlockReader2 reader = null; - private Peer peer = null; - - @Override - public RemoteBlockReader2 answer(InvocationOnMock invocation) throws Throwable { - RemoteBlockReader2 prevReader = reader; - reader = (RemoteBlockReader2) invocation.callRealMethod(); - if (peer == null) { - peer = reader.getPeer(); - } else if (prevReader != null) { - Assert.assertSame("DFSInputStream should use the same peer", - peer, reader.getPeer()); - } - return reader; - } - } - /** * (Optionally) seek to position, read and verify data. * @@ -115,33 +81,29 @@ public class TestConnCache { * @throws Exception */ @Test - @SuppressWarnings("unchecked") public void testReadFromOneDN() throws Exception { - BlockReaderTestUtil util = new BlockReaderTestUtil(1, - new HdfsConfiguration()); + HdfsConfiguration configuration = new HdfsConfiguration(); + // One of the goals of this test is to verify that we don't open more + // than one socket. So use a different client context, so that we + // get our own socket cache, rather than sharing with the other test + // instances. Also use a really long socket timeout so that nothing + // gets closed before we get around to checking the cache size at the end. + final String contextName = "testReadFromOneDNContext"; + configuration.set(DFSConfigKeys.DFS_CLIENT_CONTEXT, contextName); + configuration.setLong(DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY, + 100000000L); + BlockReaderTestUtil util = new BlockReaderTestUtil(1, configuration); final Path testFile = new Path("/testConnCache.dat"); byte authenticData[] = util.writeFile(testFile, FILE_SIZE / 1024); DFSClient client = new DFSClient( new InetSocketAddress("localhost", util.getCluster().getNameNodePort()), util.getConf()); - DFSInputStream in = Mockito.spy(client.open(testFile.toString())); + ClientContext cacheContext = + ClientContext.get(contextName, client.getConf()); + DFSInputStream in = client.open(testFile.toString()); LOG.info("opened " + testFile.toString()); byte[] dataBuf = new byte[BLOCK_SIZE]; - MockGetBlockReader answer = new MockGetBlockReader(); - Mockito.doAnswer(answer).when(in).getBlockReader( - (InetSocketAddress) Matchers.anyObject(), - (DatanodeInfo) Matchers.anyObject(), - Matchers.anyString(), - (ExtendedBlock) Matchers.anyObject(), - (Token) Matchers.anyObject(), - Matchers.anyLong(), - Matchers.anyLong(), - Matchers.anyInt(), - Matchers.anyBoolean(), - Matchers.anyString(), - (CachingStrategy)Matchers.anyObject()); - // Initial read pread(in, 0, dataBuf, 0, dataBuf.length, authenticData); // Read again and verify that the socket is the same @@ -153,5 +115,8 @@ public class TestConnCache { pread(in, 64, dataBuf, 0, dataBuf.length / 2, authenticData); in.close(); + client.close(); + Assert.assertEquals(1, + ClientContext.getFromConf(configuration).getPeerCache().size()); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferKeepalive.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferKeepalive.java index bdfc62d5fde..dc01c56d255 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferKeepalive.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferKeepalive.java @@ -22,7 +22,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SOCKET_REUSE_KEE import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CONTEXT; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -86,21 +86,22 @@ public class TestDataTransferKeepalive { // the datanode-side expiration time. final long CLIENT_EXPIRY_MS = 60000L; clientConf.setLong(DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY, CLIENT_EXPIRY_MS); - PeerCache.setInstance(DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT, CLIENT_EXPIRY_MS); + clientConf.set(DFS_CLIENT_CONTEXT, "testDatanodeRespectsKeepAliveTimeout"); DistributedFileSystem fs = (DistributedFileSystem)FileSystem.get(cluster.getURI(), clientConf); + PeerCache peerCache = ClientContext.getFromConf(clientConf).getPeerCache(); DFSTestUtil.createFile(fs, TEST_FILE, 1L, (short)1, 0L); // Clients that write aren't currently re-used. - assertEquals(0, fs.dfs.peerCache.size()); + assertEquals(0, peerCache.size()); assertXceiverCount(0); // Reads the file, so we should get a // cached socket, and should have an xceiver on the other side. DFSTestUtil.readFile(fs, TEST_FILE); - assertEquals(1, fs.dfs.peerCache.size()); + assertEquals(1, peerCache.size()); assertXceiverCount(1); // Sleep for a bit longer than the keepalive timeout @@ -111,15 +112,13 @@ public class TestDataTransferKeepalive { // The socket is still in the cache, because we don't // notice that it's closed until we try to read // from it again. - assertEquals(1, fs.dfs.peerCache.size()); + assertEquals(1, peerCache.size()); // Take it out of the cache - reading should // give an EOF. - Peer peer = fs.dfs.peerCache.get(dn.getDatanodeId(), false); + Peer peer = peerCache.get(dn.getDatanodeId(), false); assertNotNull(peer); assertEquals(-1, peer.getInputStream().read()); - PeerCache.setInstance(DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT, - DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_DEFAULT); } /** @@ -132,34 +131,33 @@ public class TestDataTransferKeepalive { // the datanode-side expiration time. final long CLIENT_EXPIRY_MS = 10L; clientConf.setLong(DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY, CLIENT_EXPIRY_MS); - PeerCache.setInstance(DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT, CLIENT_EXPIRY_MS); + clientConf.set(DFS_CLIENT_CONTEXT, "testClientResponsesKeepAliveTimeout"); DistributedFileSystem fs = (DistributedFileSystem)FileSystem.get(cluster.getURI(), clientConf); + PeerCache peerCache = ClientContext.getFromConf(clientConf).getPeerCache(); DFSTestUtil.createFile(fs, TEST_FILE, 1L, (short)1, 0L); // Clients that write aren't currently re-used. - assertEquals(0, fs.dfs.peerCache.size()); + assertEquals(0, peerCache.size()); assertXceiverCount(0); // Reads the file, so we should get a // cached socket, and should have an xceiver on the other side. DFSTestUtil.readFile(fs, TEST_FILE); - assertEquals(1, fs.dfs.peerCache.size()); + assertEquals(1, peerCache.size()); assertXceiverCount(1); // Sleep for a bit longer than the client keepalive timeout. Thread.sleep(CLIENT_EXPIRY_MS + 1); // Taking out a peer which is expired should give a null. - Peer peer = fs.dfs.peerCache.get(dn.getDatanodeId(), false); + Peer peer = peerCache.get(dn.getDatanodeId(), false); assertTrue(peer == null); // The socket cache is now empty. - assertEquals(0, fs.dfs.peerCache.size()); - PeerCache.setInstance(DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT, - DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_DEFAULT); + assertEquals(0, peerCache.size()); } /** @@ -174,7 +172,7 @@ public class TestDataTransferKeepalive { final long CLIENT_EXPIRY_MS = 600000L; Configuration clientConf = new Configuration(conf); clientConf.setLong(DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY, CLIENT_EXPIRY_MS); - PeerCache.setInstance(DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT, CLIENT_EXPIRY_MS); + clientConf.set(DFS_CLIENT_CONTEXT, "testSlowReader"); DistributedFileSystem fs = (DistributedFileSystem)FileSystem.get(cluster.getURI(), clientConf); @@ -209,7 +207,12 @@ public class TestDataTransferKeepalive { @Test(timeout=30000) public void testManyClosedSocketsInCache() throws Exception { // Make a small file - DistributedFileSystem fs = cluster.getFileSystem(); + Configuration clientConf = new Configuration(conf); + clientConf.set(DFS_CLIENT_CONTEXT, "testManyClosedSocketsInCache"); + DistributedFileSystem fs = + (DistributedFileSystem)FileSystem.get(cluster.getURI(), + clientConf); + PeerCache peerCache = ClientContext.getFromConf(clientConf).getPeerCache(); DFSTestUtil.createFile(fs, TEST_FILE, 1L, (short)1, 0L); // Insert a bunch of dead sockets in the cache, by opening @@ -227,15 +230,14 @@ public class TestDataTransferKeepalive { IOUtils.cleanup(null, stms); } - DFSClient client = ((DistributedFileSystem)fs).dfs; - assertEquals(5, client.peerCache.size()); + assertEquals(5, peerCache.size()); // Let all the xceivers timeout Thread.sleep(1500); assertXceiverCount(0); // Client side still has the sockets cached - assertEquals(5, client.peerCache.size()); + assertEquals(5, peerCache.size()); // Reading should not throw an exception. DFSTestUtil.readFile(fs, TEST_FILE); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDisableConnCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDisableConnCache.java index f7fb128bb1b..44c3f144a11 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDisableConnCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDisableConnCache.java @@ -53,7 +53,8 @@ public class TestDisableConnCache { FileSystem fsWithoutCache = FileSystem.newInstance(util.getConf()); try { DFSTestUtil.readFile(fsWithoutCache, testFile); - assertEquals(0, ((DistributedFileSystem)fsWithoutCache).dfs.peerCache.size()); + assertEquals(0, ((DistributedFileSystem)fsWithoutCache). + dfs.getClientContext().getPeerCache().size()); } finally { fsWithoutCache.close(); util.shutdown(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileInputStreamCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileInputStreamCache.java deleted file mode 100644 index 7eef5383855..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileInputStreamCache.java +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hdfs.protocol.DatanodeID; -import org.apache.hadoop.hdfs.protocol.ExtendedBlock; -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.net.unix.TemporarySocketDirectory; -import org.junit.Assert; -import org.junit.Test; - -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; - -public class TestFileInputStreamCache { - static final Log LOG = LogFactory.getLog(TestFileInputStreamCache.class); - - @Test - public void testCreateAndDestroy() throws Exception { - FileInputStreamCache cache = new FileInputStreamCache(10, 1000); - cache.close(); - } - - private static class TestFileDescriptorPair { - TemporarySocketDirectory dir = new TemporarySocketDirectory(); - FileInputStream fis[]; - - public TestFileDescriptorPair() throws IOException { - fis = new FileInputStream[2]; - for (int i = 0; i < 2; i++) { - String name = dir.getDir() + "/file" + i; - FileOutputStream fos = new FileOutputStream(name); - fos.write(1); - fos.close(); - fis[i] = new FileInputStream(name); - } - } - - public FileInputStream[] getFileInputStreams() { - return fis; - } - - public void close() throws IOException { - IOUtils.cleanup(LOG, fis); - dir.close(); - } - - public boolean compareWith(FileInputStream other[]) { - if ((other == null) || (fis == null)) { - return other == fis; - } - if (fis.length != other.length) return false; - for (int i = 0; i < fis.length; i++) { - if (fis[i] != other[i]) return false; - } - return true; - } - } - - @Test - public void testAddAndRetrieve() throws Exception { - FileInputStreamCache cache = new FileInputStreamCache(1, 1000000); - DatanodeID dnId = new DatanodeID("127.0.0.1", "localhost", - "xyzzy", 8080, 9090, 7070, 6060); - ExtendedBlock block = new ExtendedBlock("poolid", 123); - TestFileDescriptorPair pair = new TestFileDescriptorPair(); - cache.put(dnId, block, pair.getFileInputStreams()); - FileInputStream fis[] = cache.get(dnId, block); - Assert.assertTrue(pair.compareWith(fis)); - pair.close(); - cache.close(); - } - - @Test - public void testExpiry() throws Exception { - FileInputStreamCache cache = new FileInputStreamCache(1, 10); - DatanodeID dnId = new DatanodeID("127.0.0.1", "localhost", - "xyzzy", 8080, 9090, 7070, 6060); - ExtendedBlock block = new ExtendedBlock("poolid", 123); - TestFileDescriptorPair pair = new TestFileDescriptorPair(); - cache.put(dnId, block, pair.getFileInputStreams()); - Thread.sleep(cache.getExpiryTimeMs() * 100); - FileInputStream fis[] = cache.get(dnId, block); - Assert.assertNull(fis); - pair.close(); - cache.close(); - } - - @Test - public void testEviction() throws Exception { - FileInputStreamCache cache = new FileInputStreamCache(1, 10000000); - DatanodeID dnId = new DatanodeID("127.0.0.1", "localhost", - "xyzzy", 8080, 9090, 7070, 6060); - ExtendedBlock block = new ExtendedBlock("poolid", 123); - TestFileDescriptorPair pair = new TestFileDescriptorPair(); - cache.put(dnId, block, pair.getFileInputStreams()); - DatanodeID dnId2 = new DatanodeID("127.0.0.1", "localhost", - "xyzzy", 8081, 9091, 7071, 6061); - TestFileDescriptorPair pair2 = new TestFileDescriptorPair(); - cache.put(dnId2, block, pair2.getFileInputStreams()); - FileInputStream fis[] = cache.get(dnId, block); - Assert.assertNull(fis); - FileInputStream fis2[] = cache.get(dnId2, block); - Assert.assertTrue(pair2.compareWith(fis2)); - pair.close(); - cache.close(); - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java index 9c44029bfef..2644420ee1a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java @@ -303,5 +303,6 @@ public class TestFileStatus { FileSystem.LOG.info("GOOD: getting an exception", ioe); } } + fs.delete(dir, true); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitCache.java new file mode 100644 index 00000000000..ce1c2275a7d --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitCache.java @@ -0,0 +1,346 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import org.apache.commons.lang.mutable.MutableBoolean; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hdfs.client.ShortCircuitCache; +import org.apache.hadoop.hdfs.client.ShortCircuitCache.ShortCircuitReplicaCreator; +import org.apache.hadoop.hdfs.client.ShortCircuitReplica; +import org.apache.hadoop.hdfs.client.ShortCircuitReplicaInfo; +import org.apache.hadoop.hdfs.server.datanode.BlockMetadataHeader; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.net.unix.TemporarySocketDirectory; +import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.util.DataChecksum; +import org.apache.hadoop.util.Time; +import org.junit.Assert; +import org.junit.Test; + +import com.google.common.base.Preconditions; +import com.google.common.base.Supplier; + +import java.io.DataOutputStream; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; + +public class TestShortCircuitCache { + static final Log LOG = LogFactory.getLog(TestShortCircuitCache.class); + + private static class TestFileDescriptorPair { + TemporarySocketDirectory dir = new TemporarySocketDirectory(); + FileInputStream fis[]; + + public TestFileDescriptorPair() throws IOException { + fis = new FileInputStream[2]; + for (int i = 0; i < 2; i++) { + String name = dir.getDir() + "/file" + i; + FileOutputStream fos = new FileOutputStream(name); + if (i == 0) { + // write 'data' file + fos.write(1); + } else { + // write 'metadata' file + BlockMetadataHeader header = + new BlockMetadataHeader((short)1, + DataChecksum.newDataChecksum(DataChecksum.Type.NULL, 4)); + DataOutputStream dos = new DataOutputStream(fos); + BlockMetadataHeader.writeHeader(dos, header); + dos.close(); + } + fos.close(); + fis[i] = new FileInputStream(name); + } + } + + public FileInputStream[] getFileInputStreams() { + return fis; + } + + public void close() throws IOException { + IOUtils.cleanup(LOG, fis); + dir.close(); + } + + public boolean compareWith(FileInputStream data, FileInputStream meta) { + return ((data == fis[0]) && (meta == fis[1])); + } + } + + private static class SimpleReplicaCreator + implements ShortCircuitReplicaCreator { + private final int blockId; + private final ShortCircuitCache cache; + private final TestFileDescriptorPair pair; + + SimpleReplicaCreator(int blockId, ShortCircuitCache cache, + TestFileDescriptorPair pair) { + this.blockId = blockId; + this.cache = cache; + this.pair = pair; + } + + @Override + public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() { + try { + ExtendedBlockId key = new ExtendedBlockId(blockId, "test_bp1"); + return new ShortCircuitReplicaInfo( + new ShortCircuitReplica(key, + pair.getFileInputStreams()[0], pair.getFileInputStreams()[1], + cache, Time.monotonicNow())); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + @Test(timeout=60000) + public void testCreateAndDestroy() throws Exception { + ShortCircuitCache cache = + new ShortCircuitCache(10, 1, 10, 1, 1, 10000); + cache.close(); + } + + @Test(timeout=60000) + public void testAddAndRetrieve() throws Exception { + final ShortCircuitCache cache = + new ShortCircuitCache(10, 10000000, 10, 10000000, 1, 10000); + final TestFileDescriptorPair pair = new TestFileDescriptorPair(); + ShortCircuitReplicaInfo replicaInfo1 = + cache.fetchOrCreate(new ExtendedBlockId(123, "test_bp1"), + new SimpleReplicaCreator(123, cache, pair)); + Preconditions.checkNotNull(replicaInfo1.getReplica()); + Preconditions.checkState(replicaInfo1.getInvalidTokenException() == null); + pair.compareWith(replicaInfo1.getReplica().getDataStream(), + replicaInfo1.getReplica().getMetaStream()); + ShortCircuitReplicaInfo replicaInfo2 = + cache.fetchOrCreate(new ExtendedBlockId(123, "test_bp1"), + new ShortCircuitReplicaCreator() { + @Override + public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() { + Assert.fail("expected to use existing entry."); + return null; + } + }); + Preconditions.checkNotNull(replicaInfo2.getReplica()); + Preconditions.checkState(replicaInfo2.getInvalidTokenException() == null); + Preconditions.checkState(replicaInfo1 == replicaInfo2); + pair.compareWith(replicaInfo2.getReplica().getDataStream(), + replicaInfo2.getReplica().getMetaStream()); + replicaInfo1.getReplica().unref(); + replicaInfo2.getReplica().unref(); + + // Even after the reference count falls to 0, we still keep the replica + // around for a while (we have configured the expiry period to be really, + // really long here) + ShortCircuitReplicaInfo replicaInfo3 = + cache.fetchOrCreate( + new ExtendedBlockId(123, "test_bp1"), new ShortCircuitReplicaCreator() { + @Override + public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() { + Assert.fail("expected to use existing entry."); + return null; + } + }); + Preconditions.checkNotNull(replicaInfo3.getReplica()); + Preconditions.checkState(replicaInfo3.getInvalidTokenException() == null); + replicaInfo3.getReplica().unref(); + + pair.close(); + cache.close(); + } + + @Test(timeout=60000) + public void testExpiry() throws Exception { + final ShortCircuitCache cache = + new ShortCircuitCache(2, 1, 1, 10000000, 1, 10000); + final TestFileDescriptorPair pair = new TestFileDescriptorPair(); + ShortCircuitReplicaInfo replicaInfo1 = + cache.fetchOrCreate( + new ExtendedBlockId(123, "test_bp1"), new SimpleReplicaCreator(123, cache, pair)); + Preconditions.checkNotNull(replicaInfo1.getReplica()); + Preconditions.checkState(replicaInfo1.getInvalidTokenException() == null); + pair.compareWith(replicaInfo1.getReplica().getDataStream(), + replicaInfo1.getReplica().getMetaStream()); + replicaInfo1.getReplica().unref(); + final MutableBoolean triedToCreate = new MutableBoolean(false); + do { + Thread.sleep(10); + ShortCircuitReplicaInfo replicaInfo2 = + cache.fetchOrCreate( + new ExtendedBlockId(123, "test_bp1"), new ShortCircuitReplicaCreator() { + @Override + public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() { + triedToCreate.setValue(true); + return null; + } + }); + if ((replicaInfo2 != null) && (replicaInfo2.getReplica() != null)) { + replicaInfo2.getReplica().unref(); + } + } while (triedToCreate.isFalse()); + cache.close(); + } + + + @Test(timeout=60000) + public void testEviction() throws Exception { + final ShortCircuitCache cache = + new ShortCircuitCache(2, 10000000, 1, 10000000, 1, 10000); + final TestFileDescriptorPair pairs[] = new TestFileDescriptorPair[] { + new TestFileDescriptorPair(), + new TestFileDescriptorPair(), + new TestFileDescriptorPair(), + }; + ShortCircuitReplicaInfo replicaInfos[] = new ShortCircuitReplicaInfo[] { + null, + null, + null + }; + for (int i = 0; i < pairs.length; i++) { + replicaInfos[i] = cache.fetchOrCreate( + new ExtendedBlockId(i, "test_bp1"), + new SimpleReplicaCreator(i, cache, pairs[i])); + Preconditions.checkNotNull(replicaInfos[i].getReplica()); + Preconditions.checkState(replicaInfos[i].getInvalidTokenException() == null); + pairs[i].compareWith(replicaInfos[i].getReplica().getDataStream(), + replicaInfos[i].getReplica().getMetaStream()); + } + // At this point, we have 3 replicas in use. + // Let's close them all. + for (int i = 0; i < pairs.length; i++) { + replicaInfos[i].getReplica().unref(); + } + // The last two replicas should still be cached. + for (int i = 1; i < pairs.length; i++) { + final Integer iVal = new Integer(i); + replicaInfos[i] = cache.fetchOrCreate( + new ExtendedBlockId(i, "test_bp1"), + new ShortCircuitReplicaCreator() { + @Override + public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() { + Assert.fail("expected to use existing entry for " + iVal); + return null; + } + }); + Preconditions.checkNotNull(replicaInfos[i].getReplica()); + Preconditions.checkState(replicaInfos[i].getInvalidTokenException() == null); + pairs[i].compareWith(replicaInfos[i].getReplica().getDataStream(), + replicaInfos[i].getReplica().getMetaStream()); + } + // The first (oldest) replica should not be cached. + final MutableBoolean calledCreate = new MutableBoolean(false); + replicaInfos[0] = cache.fetchOrCreate( + new ExtendedBlockId(0, "test_bp1"), + new ShortCircuitReplicaCreator() { + @Override + public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() { + calledCreate.setValue(true); + return null; + } + }); + Preconditions.checkState(replicaInfos[0].getReplica() == null); + Assert.assertTrue(calledCreate.isTrue()); + // Clean up + for (int i = 1; i < pairs.length; i++) { + replicaInfos[i].getReplica().unref(); + } + for (int i = 0; i < pairs.length; i++) { + pairs[i].close(); + } + cache.close(); + } + + @Test(timeout=60000) + public void testStaleness() throws Exception { + // Set up the cache with a short staleness time. + final ShortCircuitCache cache = + new ShortCircuitCache(2, 10000000, 1, 10000000, 1, 10); + final TestFileDescriptorPair pairs[] = new TestFileDescriptorPair[] { + new TestFileDescriptorPair(), + new TestFileDescriptorPair(), + }; + ShortCircuitReplicaInfo replicaInfos[] = new ShortCircuitReplicaInfo[] { + null, + null + }; + final long HOUR_IN_MS = 60 * 60 * 1000; + for (int i = 0; i < pairs.length; i++) { + final Integer iVal = new Integer(i); + final ExtendedBlockId key = new ExtendedBlockId(i, "test_bp1"); + replicaInfos[i] = cache.fetchOrCreate(key, + new ShortCircuitReplicaCreator() { + @Override + public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() { + try { + return new ShortCircuitReplicaInfo( + new ShortCircuitReplica(key, + pairs[iVal].getFileInputStreams()[0], + pairs[iVal].getFileInputStreams()[1], + cache, Time.monotonicNow() + (iVal * HOUR_IN_MS))); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }); + Preconditions.checkNotNull(replicaInfos[i].getReplica()); + Preconditions.checkState(replicaInfos[i].getInvalidTokenException() == null); + pairs[i].compareWith(replicaInfos[i].getReplica().getDataStream(), + replicaInfos[i].getReplica().getMetaStream()); + } + + // Keep trying to getOrCreate block 0 until it goes stale (and we must re-create.) + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + ShortCircuitReplicaInfo info = cache.fetchOrCreate( + new ExtendedBlockId(0, "test_bp1"), new ShortCircuitReplicaCreator() { + @Override + public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() { + return null; + } + }); + if (info.getReplica() != null) { + info.getReplica().unref(); + return false; + } + return true; + } + }, 500, 60000); + + // Make sure that second replica did not go stale. + ShortCircuitReplicaInfo info = cache.fetchOrCreate( + new ExtendedBlockId(1, "test_bp1"), new ShortCircuitReplicaCreator() { + @Override + public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() { + Assert.fail("second replica went stale, despite 1 " + + "hour staleness time."); + return null; + } + }); + info.getReplica().unref(); + + // Clean up + for (int i = 1; i < pairs.length; i++) { + replicaInfos[i].getReplica().unref(); + } + cache.close(); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java index 57f1c117b46..4afdf62f995 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java @@ -27,6 +27,7 @@ import java.io.RandomAccessFile; import java.net.URI; import java.nio.ByteBuffer; import java.security.PrivilegedExceptionAction; +import java.util.UUID; import java.util.concurrent.TimeoutException; import org.apache.hadoop.conf.Configuration; @@ -35,8 +36,9 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.client.HdfsDataInputStream; +import org.apache.hadoop.hdfs.client.ShortCircuitCache; +import org.apache.hadoop.hdfs.client.ShortCircuitReplica; import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; -import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; @@ -125,8 +127,9 @@ public class TestShortCircuitLocalRead { throws IOException, InterruptedException { // Ensure short circuit is enabled DistributedFileSystem fs = getFileSystem(readingUser, uri, conf); + ClientContext getClientContext = ClientContext.getFromConf(conf); if (legacyShortCircuitFails) { - assertTrue(fs.getClient().useLegacyBlockReaderLocal()); + assertFalse(getClientContext.getDisableLegacyBlockReaderLocal()); } FSDataInputStream stm = fs.open(name); @@ -155,7 +158,7 @@ public class TestShortCircuitLocalRead { checkData(actual, readOffset, expected, "Read 3"); if (legacyShortCircuitFails) { - assertFalse(fs.getClient().useLegacyBlockReaderLocal()); + assertTrue(getClientContext.getDisableLegacyBlockReaderLocal()); } stm.close(); } @@ -175,8 +178,9 @@ public class TestShortCircuitLocalRead { throws IOException, InterruptedException { // Ensure short circuit is enabled DistributedFileSystem fs = getFileSystem(readingUser, uri, conf); + ClientContext clientContext = ClientContext.getFromConf(conf); if (legacyShortCircuitFails) { - assertTrue(fs.getClient().useLegacyBlockReaderLocal()); + assertTrue(clientContext.getDisableLegacyBlockReaderLocal()); } HdfsDataInputStream stm = (HdfsDataInputStream)fs.open(name); @@ -209,7 +213,7 @@ public class TestShortCircuitLocalRead { } checkData(arrayFromByteBuffer(actual), readOffset, expected, "Read 3"); if (legacyShortCircuitFails) { - assertFalse(fs.getClient().useLegacyBlockReaderLocal()); + assertTrue(clientContext.getDisableLegacyBlockReaderLocal()); } stm.close(); } @@ -223,7 +227,6 @@ public class TestShortCircuitLocalRead { public void doTestShortCircuitRead(boolean ignoreChecksum, int size, int readOffset) throws IOException, InterruptedException { - String shortCircuitUser = getCurrentUser(); doTestShortCircuitReadImpl(ignoreChecksum, size, readOffset, null, getCurrentUser(), false); } @@ -239,6 +242,10 @@ public class TestShortCircuitLocalRead { conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, true); conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_KEY, ignoreChecksum); + // Set a random client context name so that we don't share a cache with + // other invocations of this function. + conf.set(DFSConfigKeys.DFS_CLIENT_CONTEXT, + UUID.randomUUID().toString()); conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY, new File(sockDir.getDir(), "TestShortCircuitLocalRead._PORT.sock").getAbsolutePath()); @@ -322,18 +329,6 @@ public class TestShortCircuitLocalRead { doTestShortCircuitRead(true, 10*blockSize+100, 777); } - private ClientDatanodeProtocol getProxy(UserGroupInformation ugi, - final DatanodeID dnInfo, final Configuration conf) throws IOException, - InterruptedException { - return ugi.doAs(new PrivilegedExceptionAction() { - @Override - public ClientDatanodeProtocol run() throws Exception { - return DFSUtil.createClientDatanodeProtocolProxy(dnInfo, conf, 60000, - false); - } - }); - } - private static DistributedFileSystem getFileSystem(String user, final URI uri, final Configuration conf) throws InterruptedException, IOException { UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user); @@ -555,8 +550,7 @@ public class TestShortCircuitLocalRead { for (int i = 0; i < iteration; i++) { try { String user = getCurrentUser(); - checkFileContent(fs.getUri(), file1, dataToWrite, 0, user, conf, - true); + checkFileContent(fs.getUri(), file1, dataToWrite, 0, user, conf, true); } catch (IOException e) { e.printStackTrace(); } catch (InterruptedException e) { @@ -608,7 +602,8 @@ public class TestShortCircuitLocalRead { stm.write(fileData); stm.close(); try { - checkFileContent(uri, file1, fileData, readOffset, shortCircuitUser, conf, shortCircuitFails); + checkFileContent(uri, file1, fileData, readOffset, shortCircuitUser, + conf, shortCircuitFails); //RemoteBlockReader have unsupported method read(ByteBuffer bf) assertTrue("RemoteBlockReader unsupported method read(ByteBuffer bf) error", checkUnsupportedMethod(fs, file1, fileData, readOffset)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java index 53d3008dfa0..92b3d8c7934 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java @@ -38,10 +38,16 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.BlockReader; import org.apache.hadoop.hdfs.BlockReaderFactory; +import org.apache.hadoop.hdfs.ClientContext; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.DFSClient.Conf; +import org.apache.hadoop.hdfs.RemotePeerFactory; +import org.apache.hadoop.hdfs.client.ShortCircuitCache; +import org.apache.hadoop.hdfs.client.ShortCircuitReplica; +import org.apache.hadoop.hdfs.net.Peer; import org.apache.hadoop.hdfs.net.TcpPeerServer; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; @@ -55,10 +61,13 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.datanode.CachingStrategy; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.StringUtils; import org.apache.log4j.Level; +import org.junit.Assert; import org.junit.Test; public class TestBlockTokenWithDFS { @@ -131,50 +140,70 @@ public class TestBlockTokenWithDFS { } // try reading a block using a BlockReader directly - private static void tryRead(Configuration conf, LocatedBlock lblock, + private static void tryRead(final Configuration conf, LocatedBlock lblock, boolean shouldSucceed) { InetSocketAddress targetAddr = null; - Socket s = null; + IOException ioe = null; BlockReader blockReader = null; ExtendedBlock block = lblock.getBlock(); try { DatanodeInfo[] nodes = lblock.getLocations(); targetAddr = NetUtils.createSocketAddr(nodes[0].getXferAddr()); - s = NetUtils.getDefaultSocketFactory(conf).createSocket(); - s.connect(targetAddr, HdfsServerConstants.READ_TIMEOUT); - s.setSoTimeout(HdfsServerConstants.READ_TIMEOUT); - - String file = BlockReaderFactory.getFileName(targetAddr, - "test-blockpoolid", block.getBlockId()); - blockReader = BlockReaderFactory.newBlockReader( - new DFSClient.Conf(conf), file, block, lblock.getBlockToken(), 0, -1, - true, "TestBlockTokenWithDFS", TcpPeerServer.peerFromSocket(s), - nodes[0], null, null, null, false, - CachingStrategy.newDefaultStrategy()); + blockReader = new BlockReaderFactory(new DFSClient.Conf(conf)). + setFileName(BlockReaderFactory.getFileName(targetAddr, + "test-blockpoolid", block.getBlockId())). + setBlock(block). + setBlockToken(lblock.getBlockToken()). + setInetSocketAddress(targetAddr). + setStartOffset(0). + setLength(-1). + setVerifyChecksum(true). + setClientName("TestBlockTokenWithDFS"). + setDatanodeInfo(nodes[0]). + setCachingStrategy(CachingStrategy.newDefaultStrategy()). + setClientCacheContext(ClientContext.getFromConf(conf)). + setConfiguration(conf). + setRemotePeerFactory(new RemotePeerFactory() { + @Override + public Peer newConnectedPeer(InetSocketAddress addr) + throws IOException { + Peer peer = null; + Socket sock = NetUtils.getDefaultSocketFactory(conf).createSocket(); + try { + sock.connect(addr, HdfsServerConstants.READ_TIMEOUT); + sock.setSoTimeout(HdfsServerConstants.READ_TIMEOUT); + peer = TcpPeerServer.peerFromSocket(sock); + } finally { + if (peer == null) { + IOUtils.closeSocket(sock); + } + } + return peer; + } + }). + build(); } catch (IOException ex) { - if (ex instanceof InvalidBlockTokenException) { - assertFalse("OP_READ_BLOCK: access token is invalid, " - + "when it is expected to be valid", shouldSucceed); - return; - } - fail("OP_READ_BLOCK failed due to reasons other than access token: " - + StringUtils.stringifyException(ex)); + ioe = ex; } finally { - if (s != null) { + if (blockReader != null) { try { - s.close(); - } catch (IOException iex) { - } finally { - s = null; + blockReader.close(); + } catch (IOException e) { + throw new RuntimeException(e); } } } - if (blockReader == null) { - fail("OP_READ_BLOCK failed due to reasons other than access token"); + if (shouldSucceed) { + Assert.assertNotNull("OP_READ_BLOCK: access token is invalid, " + + "when it is expected to be valid", blockReader); + } else { + Assert.assertNotNull("OP_READ_BLOCK: access token is valid, " + + "when it is expected to be invalid", ioe); + Assert.assertTrue( + "OP_READ_BLOCK failed due to reasons other than access token: ", + ioe instanceof InvalidBlockTokenException); } - assertTrue("OP_READ_BLOCK: access token is valid, " - + "when it is expected to be invalid", shouldSucceed); } // get a conf for testing @@ -347,9 +376,13 @@ public class TestBlockTokenWithDFS { /* * testing READ interface on DN using a BlockReader */ - - new DFSClient(new InetSocketAddress("localhost", + DFSClient client = null; + try { + client = new DFSClient(new InetSocketAddress("localhost", cluster.getNameNodePort()), conf); + } finally { + if (client != null) client.close(); + } List locatedBlocks = nnProto.getBlockLocations( FILE_TO_READ, 0, FILE_SIZE).getLocatedBlocks(); LocatedBlock lblock = locatedBlocks.get(0); // first block diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyConsiderLoad.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyConsiderLoad.java new file mode 100644 index 00000000000..0b84fd7c953 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyConsiderLoad.java @@ -0,0 +1,161 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.blockmanagement; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.StorageType; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; +import org.apache.hadoop.hdfs.server.common.StorageInfo; +import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; +import org.apache.hadoop.test.PathUtils; +import org.apache.hadoop.util.VersionInfo; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestReplicationPolicyConsiderLoad { + + private static NameNode namenode; + private static DatanodeManager dnManager; + private static List dnrList; + private static DatanodeDescriptor[] dataNodes; + private static DatanodeStorageInfo[] storages; + + @BeforeClass + public static void setupCluster() throws IOException { + Configuration conf = new HdfsConfiguration(); + final String[] racks = { + "/rack1", + "/rack1", + "/rack1", + "/rack2", + "/rack2", + "/rack2"}; + storages = DFSTestUtil.createDatanodeStorageInfos(racks); + dataNodes = DFSTestUtil.toDatanodeDescriptor(storages); + FileSystem.setDefaultUri(conf, "hdfs://localhost:0"); + conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0"); + File baseDir = PathUtils.getTestDir(TestReplicationPolicy.class); + conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, + new File(baseDir, "name").getPath()); + conf.setBoolean( + DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY, true); + conf.setBoolean( + DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_KEY, true); + conf.setBoolean( + DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, true); + DFSTestUtil.formatNameNode(conf); + namenode = new NameNode(conf); + int blockSize = 1024; + + dnrList = new ArrayList(); + dnManager = namenode.getNamesystem().getBlockManager().getDatanodeManager(); + + // Register DNs + for (int i=0; i < 6; i++) { + DatanodeRegistration dnr = new DatanodeRegistration(dataNodes[i], + new StorageInfo(), new ExportedBlockKeys(), VersionInfo.getVersion()); + dnrList.add(dnr); + dnManager.registerDatanode(dnr); + dataNodes[i].getStorageInfos()[0].setUtilizationForTesting( + 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*blockSize, 0L, + 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*blockSize, 0L); + dataNodes[i].updateHeartbeat( + BlockManagerTestUtil.getStorageReportsForDatanode(dataNodes[i]), + 0L, 0L, 0, 0); + } + } + + /** + * Tests that chooseTarget with considerLoad set to true correctly calculates + * load with decommissioned nodes. + */ + @Test + public void testChooseTargetWithDecomNodes() throws IOException { + namenode.getNamesystem().writeLock(); + try { + // Decommission DNs so BlockPlacementPolicyDefault.isGoodTarget() + // returns false + for (int i = 0; i < 3; i++) { + DatanodeInfo d = dnManager.getDatanodeByXferAddr( + dnrList.get(i).getIpAddr(), + dnrList.get(i).getXferPort()); + d.setDecommissioned(); + } + String blockPoolId = namenode.getNamesystem().getBlockPoolId(); + dnManager.handleHeartbeat(dnrList.get(3), + BlockManagerTestUtil.getStorageReportsForDatanode(dataNodes[3]), + blockPoolId, dataNodes[3].getCacheCapacity(), + dataNodes[3].getCacheRemaining(), + 2, 0, 0); + dnManager.handleHeartbeat(dnrList.get(4), + BlockManagerTestUtil.getStorageReportsForDatanode(dataNodes[4]), + blockPoolId, dataNodes[4].getCacheCapacity(), + dataNodes[4].getCacheRemaining(), + 4, 0, 0); + dnManager.handleHeartbeat(dnrList.get(5), + BlockManagerTestUtil.getStorageReportsForDatanode(dataNodes[5]), + blockPoolId, dataNodes[5].getCacheCapacity(), + dataNodes[5].getCacheRemaining(), + 4, 0, 0); + + // Call chooseTarget() + DatanodeStorageInfo[] targets = namenode.getNamesystem().getBlockManager() + .getBlockPlacementPolicy().chooseTarget("testFile.txt", 3, + dataNodes[0], new ArrayList(), false, null, + 1024, StorageType.DEFAULT); + + assertEquals(3, targets.length); + Set targetSet = new HashSet( + Arrays.asList(targets)); + for (int i = 3; i < storages.length; i++) { + assertTrue(targetSet.contains(storages[i])); + } + } finally { + dataNodes[0].stopDecommission(); + dataNodes[1].stopDecommission(); + dataNodes[2].stopDecommission(); + namenode.getNamesystem().writeUnlock(); + } + NameNode.LOG.info("Done working on it"); + } + + @AfterClass + public static void teardownCluster() { + if (namenode != null) namenode.stop(); + } + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java index 646d33d13b8..f5b31847dc1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java @@ -35,11 +35,14 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.BlockReader; import org.apache.hadoop.hdfs.BlockReaderFactory; +import org.apache.hadoop.hdfs.ClientContext; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.RemotePeerFactory; +import org.apache.hadoop.hdfs.net.Peer; import org.apache.hadoop.hdfs.net.TcpPeerServer; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; @@ -48,13 +51,14 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; -import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.UserGroupInformation; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -284,23 +288,43 @@ public class TestDataNodeVolumeFailure { private void accessBlock(DatanodeInfo datanode, LocatedBlock lblock) throws IOException { InetSocketAddress targetAddr = null; - Socket s = null; ExtendedBlock block = lblock.getBlock(); targetAddr = NetUtils.createSocketAddr(datanode.getXferAddr()); - - s = NetUtils.getDefaultSocketFactory(conf).createSocket(); - s.connect(targetAddr, HdfsServerConstants.READ_TIMEOUT); - s.setSoTimeout(HdfsServerConstants.READ_TIMEOUT); - String file = BlockReaderFactory.getFileName(targetAddr, - "test-blockpoolid", - block.getBlockId()); - BlockReader blockReader = - BlockReaderFactory.newBlockReader(new DFSClient.Conf(conf), file, block, - lblock.getBlockToken(), 0, -1, true, "TestDataNodeVolumeFailure", - TcpPeerServer.peerFromSocket(s), datanode, null, null, null, false, - CachingStrategy.newDefaultStrategy()); + BlockReader blockReader = new BlockReaderFactory(new DFSClient.Conf(conf)). + setInetSocketAddress(targetAddr). + setBlock(block). + setFileName(BlockReaderFactory.getFileName(targetAddr, + "test-blockpoolid", block.getBlockId())). + setBlockToken(lblock.getBlockToken()). + setStartOffset(0). + setLength(-1). + setVerifyChecksum(true). + setClientName("TestDataNodeVolumeFailure"). + setDatanodeInfo(datanode). + setCachingStrategy(CachingStrategy.newDefaultStrategy()). + setClientCacheContext(ClientContext.getFromConf(conf)). + setConfiguration(conf). + setRemotePeerFactory(new RemotePeerFactory() { + @Override + public Peer newConnectedPeer(InetSocketAddress addr) + throws IOException { + Peer peer = null; + Socket sock = NetUtils.getDefaultSocketFactory(conf).createSocket(); + try { + sock.connect(addr, HdfsServerConstants.READ_TIMEOUT); + sock.setSoTimeout(HdfsServerConstants.READ_TIMEOUT); + peer = TcpPeerServer.peerFromSocket(sock); + } finally { + if (peer == null) { + IOUtils.closeSocket(sock); + } + } + return peer; + } + }). + build(); blockReader.close(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeduplicationMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeduplicationMap.java new file mode 100644 index 00000000000..447c7ebd0e5 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeduplicationMap.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.server.namenode; + +import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.SaverContext.DeduplicationMap; +import org.junit.Assert; +import org.junit.Test; + +public class TestDeduplicationMap { + @Test + public void testDeduplicationMap() { + DeduplicationMap m = DeduplicationMap.newMap(); + Assert.assertEquals(1, m.getId("1")); + Assert.assertEquals(2, m.getId("2")); + Assert.assertEquals(3, m.getId("3")); + Assert.assertEquals(1, m.getId("1")); + Assert.assertEquals(2, m.getId("2")); + Assert.assertEquals(3, m.getId("3")); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java new file mode 100644 index 00000000000..552b091b7b4 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java @@ -0,0 +1,138 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.util.EnumSet; + +import junit.framework.Assert; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSOutputStream; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag; +import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; +import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease; +import org.apache.hadoop.hdfs.util.MD5FileUtils; +import org.junit.Test; + +public class TestFSImage { + + @Test + public void testPersist() throws IOException { + Configuration conf = new Configuration(); + testPersistHelper(conf); + } + + @Test + public void testCompression() throws IOException { + Configuration conf = new Configuration(); + conf.setBoolean(DFSConfigKeys.DFS_IMAGE_COMPRESS_KEY, true); + conf.set(DFSConfigKeys.DFS_IMAGE_COMPRESSION_CODEC_KEY, + "org.apache.hadoop.io.compress.GzipCodec"); + testPersistHelper(conf); + } + + private void testPersistHelper(Configuration conf) throws IOException { + MiniDFSCluster cluster = null; + try { + cluster = new MiniDFSCluster.Builder(conf).build(); + cluster.waitActive(); + FSNamesystem fsn = cluster.getNamesystem(); + DistributedFileSystem fs = cluster.getFileSystem(); + + final Path dir = new Path("/abc/def"); + final Path file1 = new Path(dir, "f1"); + final Path file2 = new Path(dir, "f2"); + + // create an empty file f1 + fs.create(file1).close(); + + // create an under-construction file f2 + FSDataOutputStream out = fs.create(file2); + out.writeBytes("hello"); + ((DFSOutputStream) out.getWrappedStream()).hsync(EnumSet + .of(SyncFlag.UPDATE_LENGTH)); + + // checkpoint + fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + fs.saveNamespace(); + fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + + cluster.restartNameNode(); + cluster.waitActive(); + fs = cluster.getFileSystem(); + + assertTrue(fs.isDirectory(dir)); + assertTrue(fs.exists(file1)); + assertTrue(fs.exists(file2)); + + // check internals of file2 + INodeFile file2Node = fsn.dir.getINode4Write(file2.toString()).asFile(); + assertEquals("hello".length(), file2Node.computeFileSize()); + assertTrue(file2Node.isUnderConstruction()); + BlockInfo[] blks = file2Node.getBlocks(); + assertEquals(1, blks.length); + assertEquals(BlockUCState.UNDER_CONSTRUCTION, blks[0].getBlockUCState()); + // check lease manager + Lease lease = fsn.leaseManager.getLeaseByPath(file2.toString()); + Assert.assertNotNull(lease); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } + + /** + * Ensure that the digest written by the saver equals to the digest of the + * file. + */ + @Test + public void testDigest() throws IOException { + Configuration conf = new Configuration(); + MiniDFSCluster cluster = null; + try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build(); + DistributedFileSystem fs = cluster.getFileSystem(); + fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + fs.saveNamespace(); + fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + File currentDir = FSImageTestUtil.getNameNodeCurrentDirs(cluster, 0).get( + 0); + File fsimage = FSImageTestUtil.findNewestImageFile(currentDir + .getAbsolutePath()); + assertEquals(MD5FileUtils.readStoredMd5ForFile(fsimage), + MD5FileUtils.computeMd5ForFile(fsimage)); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java index 5e3ac4b7a2b..bb03b30c860 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java @@ -27,17 +27,12 @@ import static org.junit.Assert.assertTrue; import java.io.File; import java.io.IOException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.FSImageFile; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; import org.junit.Test; public class TestFSImageStorageInspector { - private static final Log LOG = LogFactory.getLog( - TestFSImageStorageInspector.class); - /** * Simple test with image, edits, and inprogress edits */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java index 21935d05d9c..f3cbf15aae2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java @@ -140,7 +140,7 @@ public class TestFSImageWithSnapshot { private File saveFSImageToTempFile() throws IOException { SaveNamespaceContext context = new SaveNamespaceContext(fsn, txid, new Canceler()); - FSImageFormat.Saver saver = new FSImageFormat.Saver(context); + FSImageFormatProtobuf.Saver saver = new FSImageFormatProtobuf.Saver(context); FSImageCompression compression = FSImageCompression.createCompression(conf); File imageFile = getImageFile(testDir, txid); fsn.readLock(); @@ -154,7 +154,7 @@ public class TestFSImageWithSnapshot { /** Load the fsimage from a temp file */ private void loadFSImageFromTempFile(File imageFile) throws IOException { - FSImageFormat.Loader loader = new FSImageFormat.Loader(conf, fsn); + FSImageFormat.LoaderDelegator loader = FSImageFormat.newLoader(conf, fsn); fsn.writeLock(); fsn.getFSDirectory().writeLock(); try { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java index e27eca375c2..8a8e0736211 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java @@ -288,7 +288,6 @@ public class TestStandbyCheckpoints { doEdits(0, 1000); nn0.getRpcServer().rollEditLog(); answerer.waitForCall(); - answerer.proceed(); assertTrue("SBN is not performing checkpoint but it should be.", answerer.getFireCount() == 1 && answerer.getResultCount() == 0); @@ -307,6 +306,7 @@ public class TestStandbyCheckpoints { // RPC to the SBN happened during the checkpoint. assertTrue("SBN should have still been checkpointing.", answerer.getFireCount() == 1 && answerer.getResultCount() == 0); + answerer.proceed(); answerer.waitForResult(); assertTrue("SBN should have finished checkpointing.", answerer.getFireCount() == 1 && answerer.getResultCount() == 1); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java index 7fe8087f2a4..d4e887949e0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java @@ -73,7 +73,6 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; -; /** Testing rename with snapshots. */ public class TestRenameWithSnapshots { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java index 27228bd0482..20cc1351e8d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java @@ -25,6 +25,9 @@ import static org.junit.Assert.fail; import java.io.File; import java.io.IOException; +import java.io.PrintWriter; +import java.io.RandomAccessFile; +import java.io.StringWriter; import java.util.ArrayList; import java.util.Arrays; import java.util.EnumSet; @@ -53,8 +56,7 @@ import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper.TestDirectoryTree; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper.TestDirectoryTree.Node; -import org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer; -import org.apache.hadoop.hdfs.tools.offlineImageViewer.XmlImageVisitor; +import org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Time; @@ -245,8 +247,8 @@ public class TestSnapshot { * snapshots */ @Test - public void testOfflineImageViewer() throws Throwable { - runTestSnapshot(SNAPSHOT_ITERATION_NUMBER); + public void testOfflineImageViewer() throws Exception { + runTestSnapshot(1); // retrieve the fsimage. Note that we already save namespace to fsimage at // the end of each iteration of runTestSnapshot. @@ -254,31 +256,10 @@ public class TestSnapshot { FSImageTestUtil.getFSImage( cluster.getNameNode()).getStorage().getStorageDir(0)); assertNotNull("Didn't generate or can't find fsimage", originalFsimage); - - String ROOT = System.getProperty("test.build.data", "build/test/data"); - File testFile = new File(ROOT, "/image"); - String xmlImage = ROOT + "/image_xml"; - boolean success = false; - - try { - DFSTestUtil.copyFile(originalFsimage, testFile); - XmlImageVisitor v = new XmlImageVisitor(xmlImage, true); - OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, - true); - oiv.go(); - success = true; - } finally { - if (testFile.exists()) { - testFile.delete(); - } - // delete the xml file if the parsing is successful - if (success) { - File xmlImageFile = new File(xmlImage); - if (xmlImageFile.exists()) { - xmlImageFile.delete(); - } - } - } + StringWriter output = new StringWriter(); + PrintWriter o = new PrintWriter(output); + PBImageXmlWriter v = new PBImageXmlWriter(new Configuration(), o); + v.visit(new RandomAccessFile(originalFsimage, "r")); } private void runTestSnapshot(int iteration) throws Exception { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java index fb8d5292844..9fe3deab1bd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java @@ -92,7 +92,7 @@ public class TestWebHdfsDataLocality { //The chosen datanode must be the same as the client address final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode( - namenode, f, PutOpParam.Op.CREATE, -1L, blocksize, conf); + namenode, f, PutOpParam.Op.CREATE, -1L, blocksize); Assert.assertEquals(ipAddr, chosen.getIpAddr()); } } @@ -117,19 +117,19 @@ public class TestWebHdfsDataLocality { { //test GETFILECHECKSUM final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode( - namenode, f, GetOpParam.Op.GETFILECHECKSUM, -1L, blocksize, conf); + namenode, f, GetOpParam.Op.GETFILECHECKSUM, -1L, blocksize); Assert.assertEquals(expected, chosen); } { //test OPEN final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode( - namenode, f, GetOpParam.Op.OPEN, 0, blocksize, conf); + namenode, f, GetOpParam.Op.OPEN, 0, blocksize); Assert.assertEquals(expected, chosen); } { //test APPEND final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode( - namenode, f, PostOpParam.Op.APPEND, -1L, blocksize, conf); + namenode, f, PostOpParam.Op.APPEND, -1L, blocksize); Assert.assertEquals(expected, chosen); } } finally { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java index 11aa3b821f0..91a5c1521c7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java @@ -20,23 +20,20 @@ package org.apache.hadoop.hdfs.tools.offlineImageViewer; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; import java.io.BufferedReader; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.EOFException; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; +import java.io.PrintWriter; +import java.io.RandomAccessFile; +import java.io.StringWriter; import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -46,27 +43,29 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.test.PathUtils; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; - +import org.junit.rules.TemporaryFolder; /** - * Test function of OfflineImageViewer by: - * * confirming it can correctly process a valid fsimage file and that - * the processing generates a correct representation of the namespace - * * confirming it correctly fails to process an fsimage file with a layout - * version it shouldn't be able to handle - * * confirm it correctly bails on malformed image files, in particular, a - * file that ends suddenly. + * Test function of OfflineImageViewer by: * confirming it can correctly process + * a valid fsimage file and that the processing generates a correct + * representation of the namespace * confirming it correctly fails to process an + * fsimage file with a layout version it shouldn't be able to handle * confirm + * it correctly bails on malformed image files, in particular, a file that ends + * suddenly. */ public class TestOfflineImageViewer { private static final Log LOG = LogFactory.getLog(OfflineImageViewer.class); @@ -76,22 +75,22 @@ public class TestOfflineImageViewer { private static File originalFsimage = null; // Elements of lines of ls-file output to be compared to FileStatus instance - private static class LsElements { - public String perms; - public int replication; - public String username; - public String groupname; - public long filesize; - public char dir; // d if dir, - otherwise + private static final class LsElements { + private String perms; + private int replication; + private String username; + private String groupname; + private long filesize; + private boolean isDir; } - + // namespace as written to dfs, to be compared with viewer's output - final static HashMap writtenFiles = - new HashMap(); - - private static String ROOT = PathUtils.getTestDirName(TestOfflineImageViewer.class); - - // Create a populated namespace for later testing. Save its contents to a + final static HashMap writtenFiles = new HashMap(); + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + // Create a populated namespace for later testing. Save its contents to a // data structure and store its fsimage location. // We only want to generate the fsimage file once and use it for // multiple tests. @@ -100,35 +99,39 @@ public class TestOfflineImageViewer { MiniDFSCluster cluster = null; try { Configuration conf = new HdfsConfiguration(); - conf.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, 10000); - conf.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, 5000); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); + conf.setLong( + DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, 10000); + conf.setLong( + DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, 5000); + conf.setBoolean( + DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL, "RULE:[2:$1@$0](JobTracker@.*FOO.COM)s/@.*//" + "DEFAULT"); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); cluster.waitActive(); FileSystem hdfs = cluster.getFileSystem(); - + int filesize = 256; - - // Create a reasonable namespace - for(int i = 0; i < NUM_DIRS; i++) { + + // Create a reasonable namespace + for (int i = 0; i < NUM_DIRS; i++) { Path dir = new Path("/dir" + i); hdfs.mkdirs(dir); writtenFiles.put(dir.toString(), pathToFileEntry(hdfs, dir.toString())); - for(int j = 0; j < FILES_PER_DIR; j++) { + for (int j = 0; j < FILES_PER_DIR; j++) { Path file = new Path(dir, "file" + j); FSDataOutputStream o = hdfs.create(file); - o.write(new byte[ filesize++ ]); + o.write(new byte[filesize++]); o.close(); - - writtenFiles.put(file.toString(), pathToFileEntry(hdfs, file.toString())); + + writtenFiles.put(file.toString(), + pathToFileEntry(hdfs, file.toString())); } } // Get delegation tokens so we log the delegation token op - Token[] delegationTokens = - hdfs.addDelegationTokens(TEST_RENEWER, null); + Token[] delegationTokens = hdfs + .addDelegationTokens(TEST_RENEWER, null); for (Token t : delegationTokens) { LOG.debug("got token " + t); } @@ -137,329 +140,113 @@ public class TestOfflineImageViewer { cluster.getNameNodeRpc() .setSafeMode(SafeModeAction.SAFEMODE_ENTER, false); cluster.getNameNodeRpc().saveNamespace(); - + // Determine location of fsimage file - originalFsimage = FSImageTestUtil.findLatestImageFile( - FSImageTestUtil.getFSImage( - cluster.getNameNode()).getStorage().getStorageDir(0)); + originalFsimage = FSImageTestUtil.findLatestImageFile(FSImageTestUtil + .getFSImage(cluster.getNameNode()).getStorage().getStorageDir(0)); if (originalFsimage == null) { throw new RuntimeException("Didn't generate or can't find fsimage"); } LOG.debug("original FS image file is " + originalFsimage); } finally { - if(cluster != null) + if (cluster != null) cluster.shutdown(); } } - + @AfterClass public static void deleteOriginalFSImage() throws IOException { - if(originalFsimage != null && originalFsimage.exists()) { + if (originalFsimage != null && originalFsimage.exists()) { originalFsimage.delete(); } } - - // Convenience method to generate a file status from file system for + + // Convenience method to generate a file status from file system for // later comparison - private static FileStatus pathToFileEntry(FileSystem hdfs, String file) - throws IOException { + private static FileStatus pathToFileEntry(FileSystem hdfs, String file) + throws IOException { return hdfs.getFileStatus(new Path(file)); } - - // Verify that we can correctly generate an ls-style output for a valid + + // Verify that we can correctly generate an ls-style output for a valid // fsimage @Test public void outputOfLSVisitor() throws IOException { - File testFile = new File(ROOT, "/basicCheck"); - File outputFile = new File(ROOT, "/basicCheckOutput"); - - try { - DFSTestUtil.copyFile(originalFsimage, testFile); - - ImageVisitor v = new LsImageVisitor(outputFile.getPath(), true); - OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, false); - - oiv.go(); - - HashMap fileOutput = readLsfile(outputFile); - - compareNamespaces(writtenFiles, fileOutput); - } finally { - if(testFile.exists()) testFile.delete(); - if(outputFile.exists()) outputFile.delete(); - } - LOG.debug("Correctly generated ls-style output."); - } - - // Confirm that attempting to read an fsimage file with an unsupported - // layout results in an error - @Test - public void unsupportedFSLayoutVersion() throws IOException { - File testFile = new File(ROOT, "/invalidLayoutVersion"); - File outputFile = new File(ROOT, "invalidLayoutVersionOutput"); - - try { - int badVersionNum = -432; - changeLayoutVersion(originalFsimage, testFile, badVersionNum); - ImageVisitor v = new LsImageVisitor(outputFile.getPath(), true); - OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, false); - - try { - oiv.go(); - fail("Shouldn't be able to read invalid laytout version"); - } catch(IOException e) { - if(!e.getMessage().contains(Integer.toString(badVersionNum))) - throw e; // wasn't error we were expecting - LOG.debug("Correctly failed at reading bad image version."); + StringWriter output = new StringWriter(); + PrintWriter out = new PrintWriter(output); + LsrPBImage v = new LsrPBImage(new Configuration(), out); + v.visit(new RandomAccessFile(originalFsimage, "r")); + out.close(); + Pattern pattern = Pattern + .compile("([d\\-])([rwx\\-]{9})\\s*(-|\\d+)\\s*(\\w+)\\s*(\\w+)\\s*(\\d+)\\s*(\\d+)\\s*([\b/]+)"); + int count = 0; + for (String s : output.toString().split("\n")) { + Matcher m = pattern.matcher(s); + assertTrue(m.find()); + LsElements e = new LsElements(); + e.isDir = m.group(1).equals("d"); + e.perms = m.group(2); + e.replication = m.group(3).equals("-") ? 0 : Integer.parseInt(m.group(3)); + e.username = m.group(4); + e.groupname = m.group(5); + e.filesize = Long.parseLong(m.group(7)); + String path = m.group(8); + if (!path.equals("/")) { + compareFiles(writtenFiles.get(path), e); } - } finally { - if(testFile.exists()) testFile.delete(); - if(outputFile.exists()) outputFile.delete(); + ++count; } + assertEquals(writtenFiles.size() + 1, count); } - - // Verify that image viewer will bail on a file that ends unexpectedly - @Test - public void truncatedFSImage() throws IOException { - File testFile = new File(ROOT, "/truncatedFSImage"); - File outputFile = new File(ROOT, "/trucnatedFSImageOutput"); - try { - copyPartOfFile(originalFsimage, testFile); - assertTrue("Created truncated fsimage", testFile.exists()); - - ImageVisitor v = new LsImageVisitor(outputFile.getPath(), true); - OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, false); - try { - oiv.go(); - fail("Managed to process a truncated fsimage file"); - } catch (EOFException e) { - LOG.debug("Correctly handled EOF"); - } - - } finally { - if(testFile.exists()) testFile.delete(); - if(outputFile.exists()) outputFile.delete(); - } + @Test(expected = IOException.class) + public void testTruncatedFSImage() throws IOException { + File truncatedFile = folder.newFile(); + StringWriter output = new StringWriter(); + copyPartOfFile(originalFsimage, truncatedFile); + new FileDistributionCalculator(new Configuration(), 0, 0, new PrintWriter( + output)).visit(new RandomAccessFile(truncatedFile, "r")); } - - // Test that our ls file has all the same compenents of the original namespace - private void compareNamespaces(HashMap written, - HashMap fileOutput) { - assertEquals( "Should be the same number of files in both, plus one for root" - + " in fileoutput", fileOutput.keySet().size(), - written.keySet().size() + 1); - Set inFile = fileOutput.keySet(); - // For each line in the output file, verify that the namespace had a - // filestatus counterpart - for (String path : inFile) { - if (path.equals("/")) // root's not included in output from system call - continue; - - assertTrue("Path in file (" + path + ") was written to fs", written - .containsKey(path)); - - compareFiles(written.get(path), fileOutput.get(path)); - - written.remove(path); - } - - assertEquals("No more files were written to fs", 0, written.size()); - } - // Compare two files as listed in the original namespace FileStatus and // the output of the ls file from the image processor private void compareFiles(FileStatus fs, LsElements elements) { - assertEquals("directory listed as such", - fs.isDirectory() ? 'd' : '-', elements.dir); - assertEquals("perms string equal", - fs.getPermission().toString(), elements.perms); + assertEquals("directory listed as such", fs.isDirectory(), elements.isDir); + assertEquals("perms string equal", fs.getPermission().toString(), + elements.perms); assertEquals("replication equal", fs.getReplication(), elements.replication); assertEquals("owner equal", fs.getOwner(), elements.username); assertEquals("group equal", fs.getGroup(), elements.groupname); assertEquals("lengths equal", fs.getLen(), elements.filesize); } - // Read the contents of the file created by the Ls processor - private HashMap readLsfile(File lsFile) throws IOException { - BufferedReader br = new BufferedReader(new FileReader(lsFile)); - String line = null; - HashMap fileContents = new HashMap(); - - while((line = br.readLine()) != null) - readLsLine(line, fileContents); - - br.close(); - return fileContents; - } - - // Parse a line from the ls output. Store permissions, replication, - // username, groupname and filesize in hashmap keyed to the path name - private void readLsLine(String line, HashMap fileContents) { - String elements [] = line.split("\\s+"); - - assertEquals("Not enough elements in ls output", 8, elements.length); - - LsElements lsLine = new LsElements(); - - lsLine.dir = elements[0].charAt(0); - lsLine.perms = elements[0].substring(1); - lsLine.replication = elements[1].equals("-") - ? 0 : Integer.valueOf(elements[1]); - lsLine.username = elements[2]; - lsLine.groupname = elements[3]; - lsLine.filesize = Long.valueOf(elements[4]); - // skipping date and time - - String path = elements[7]; - - // Check that each file in the ls output was listed once - assertFalse("LS file had duplicate file entries", - fileContents.containsKey(path)); - - fileContents.put(path, lsLine); - } - - // Copy one fsimage to another, changing the layout version in the process - private void changeLayoutVersion(File src, File dest, int newVersion) - throws IOException { - DataInputStream in = null; - DataOutputStream out = null; - - try { - in = new DataInputStream(new FileInputStream(src)); - out = new DataOutputStream(new FileOutputStream(dest)); - - in.readInt(); - out.writeInt(newVersion); - - byte [] b = new byte[1024]; - while( in.read(b) > 0 ) { - out.write(b); - } - } finally { - if(in != null) in.close(); - if(out != null) out.close(); - } - } - - // Only copy part of file into the other. Used for testing truncated fsimage private void copyPartOfFile(File src, File dest) throws IOException { - InputStream in = null; - OutputStream out = null; - - byte [] b = new byte[256]; - int bytesWritten = 0; - int count; - int maxBytes = 700; - + FileInputStream in = null; + FileOutputStream out = null; + final int MAX_BYTES = 700; try { in = new FileInputStream(src); out = new FileOutputStream(dest); - - while( (count = in.read(b)) > 0 && bytesWritten < maxBytes ) { - out.write(b); - bytesWritten += count; - } + in.getChannel().transferTo(0, MAX_BYTES, out.getChannel()); } finally { - if(in != null) in.close(); - if(out != null) out.close(); + IOUtils.cleanup(null, in); + IOUtils.cleanup(null, out); } } @Test - public void outputOfFileDistributionVisitor() throws IOException { - File testFile = new File(ROOT, "/basicCheck"); - File outputFile = new File(ROOT, "/fileDistributionCheckOutput"); + public void testFileDistributionVisitor() throws IOException { + StringWriter output = new StringWriter(); + PrintWriter o = new PrintWriter(output); + new FileDistributionCalculator(new Configuration(), 0, 0, o) + .visit(new RandomAccessFile(originalFsimage, "r")); + o.close(); - int totalFiles = 0; - BufferedReader reader = null; - try { - DFSTestUtil.copyFile(originalFsimage, testFile); - ImageVisitor v = new FileDistributionVisitor(outputFile.getPath(), 0, 0); - OfflineImageViewer oiv = - new OfflineImageViewer(testFile.getPath(), v, false); + Pattern p = Pattern.compile("totalFiles = (\\d+)\n"); + Matcher matcher = p.matcher(output.getBuffer()); - oiv.go(); - - reader = new BufferedReader(new FileReader(outputFile)); - String line = reader.readLine(); - assertEquals(line, "Size\tNumFiles"); - while((line = reader.readLine()) != null) { - String[] row = line.split("\t"); - assertEquals(row.length, 2); - totalFiles += Integer.parseInt(row[1]); - } - } finally { - if (reader != null) { - reader.close(); - } - if(testFile.exists()) testFile.delete(); - if(outputFile.exists()) outputFile.delete(); - } + assertTrue(matcher.find() && matcher.groupCount() == 1); + int totalFiles = Integer.parseInt(matcher.group(1)); assertEquals(totalFiles, NUM_DIRS * FILES_PER_DIR); } - - private static class TestImageVisitor extends ImageVisitor { - private List delegationTokenRenewers = new LinkedList(); - TestImageVisitor() { - } - - List getDelegationTokenRenewers() { - return delegationTokenRenewers; - } - - @Override - void start() throws IOException { - } - - @Override - void finish() throws IOException { - } - - @Override - void finishAbnormally() throws IOException { - } - - @Override - void visit(ImageElement element, String value) throws IOException { - if (element == ImageElement.DELEGATION_TOKEN_IDENTIFIER_RENEWER) { - delegationTokenRenewers.add(value); - } - } - - @Override - void visitEnclosingElement(ImageElement element) throws IOException { - } - - @Override - void visitEnclosingElement(ImageElement element, ImageElement key, - String value) throws IOException { - } - - @Override - void leaveEnclosingElement() throws IOException { - } - } - - @Test - public void outputOfTestVisitor() throws IOException { - File testFile = new File(ROOT, "/basicCheck"); - - try { - DFSTestUtil.copyFile(originalFsimage, testFile); - TestImageVisitor v = new TestImageVisitor(); - OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, true); - oiv.go(); - - // Validated stored delegation token identifiers. - List dtrs = v.getDelegationTokenRenewers(); - assertEquals(1, dtrs.size()); - assertEquals(TEST_RENEWER, dtrs.get(0)); - } finally { - if(testFile.exists()) testFile.delete(); - } - LOG.debug("Passed TestVisitor validation."); - } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHftpFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHftpFileSystem.java index 06ac50a0c7f..db0fda5e743 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHftpFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHftpFileSystem.java @@ -136,6 +136,7 @@ public class TestHftpFileSystem { out.close(); FSDataInputStream in = hftpFs.open(p); assertEquals('0', in.read()); + in.close(); // Check the file status matches the path. Hftp returns a FileStatus // with the entire URI, extract the path part. @@ -250,6 +251,7 @@ public class TestHftpFileSystem { FSDataInputStream in = hftpFs.open(testFile); in.seek(7); assertEquals('7', in.read()); + in.close(); } @Test diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java index c4f30b3ebf2..0942ef26726 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.web; import java.io.File; +import java.io.InputStream; import java.net.InetSocketAddress; import java.net.URI; @@ -92,6 +93,9 @@ public class TestHttpsFileSystem { os.write(23); os.close(); Assert.assertTrue(fs.exists(f)); + InputStream is = fs.open(f); + Assert.assertEquals(23, is.read()); + is.close(); fs.close(); } } diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 3b72f402223..02099b7ce7a 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -158,8 +158,26 @@ Release 2.4.0 - UNRELEASED OPTIMIZATIONS BUG FIXES + + MAPREDUCE-5746. Job diagnostics can implicate wrong task for a failed job. + (Jason Lowe via kasha) -Release 2.3.0 - UNRELEASED + MAPREDUCE-5670. CombineFileRecordReader should report progress when moving + to the next file (Chen He via jlowe) + +Release 2.3.1 - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + +Release 2.3.0 - 2014-02-18 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/CombineFileRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/CombineFileRecordReader.java index 1abaef260c5..f54f1760d7d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/CombineFileRecordReader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/CombineFileRecordReader.java @@ -140,6 +140,8 @@ public class CombineFileRecordReader implements RecordReader { return false; } + reporter.progress(); + // get a record reader for the idx-th chunk try { curReader = rrConstructor.newInstance(new Object [] diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java index 9d6f579c44d..19e2a51a132 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java @@ -353,8 +353,10 @@ public class JobHistoryParser implements HistoryEventHandler { taskInfo.error = StringInterner.weakIntern(event.getError()); taskInfo.failedDueToAttemptId = event.getFailedAttemptID(); taskInfo.counters = event.getCounters(); - info.errorInfo = "Task " + taskInfo.taskId +" failed " + - taskInfo.attemptsMap.size() + " times "; + if (info.errorInfo.isEmpty()) { + info.errorInfo = "Task " + taskInfo.taskId + " failed " + + taskInfo.attemptsMap.size() + " times "; + } } private void handleTaskStartedEvent(TaskStartedEvent event) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileRecordReader.java index fb86cbafc12..767f79a1c02 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileRecordReader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileRecordReader.java @@ -54,7 +54,7 @@ public class CombineFileRecordReader extends RecordReader { protected int idx; protected long progress; protected RecordReader curReader; - + public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { this.split = (CombineFileSplit)split; @@ -144,6 +144,8 @@ public class CombineFileRecordReader extends RecordReader { return false; } + context.progress(); + // get a record reader for the idx-th chunk try { Configuration conf = context.getConfiguration(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/lib/TestCombineFileRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/lib/TestCombineFileRecordReader.java new file mode 100644 index 00000000000..296aa232389 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/lib/TestCombineFileRecordReader.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapred.lib; + +import java.io.File; +import java.io.FileWriter; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.TextInputFormat; +import org.apache.hadoop.fs.FileUtil; + +import org.junit.Test; +import org.mockito.Mockito; +import org.junit.Assert; + +import java.io.IOException; + +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class TestCombineFileRecordReader { + + private static Path outDir = new Path(System.getProperty("test.build.data", + "/tmp"), TestCombineFileRecordReader.class.getName()); + + private static class TextRecordReaderWrapper + extends org.apache.hadoop.mapred.lib.CombineFileRecordReaderWrapper { + // this constructor signature is required by CombineFileRecordReader + public TextRecordReaderWrapper(CombineFileSplit split, Configuration conf, + Reporter reporter, Integer idx) throws IOException { + super(new TextInputFormat(), split, conf, reporter, idx); + } + } + + @SuppressWarnings("unchecked") + @Test + public void testInitNextRecordReader() throws IOException{ + JobConf conf = new JobConf(); + Path[] paths = new Path[3]; + long[] fileLength = new long[3]; + File[] files = new File[3]; + LongWritable key = new LongWritable(1); + Text value = new Text(); + try { + for(int i=0;i<3;i++){ + fileLength[i] = i; + File dir = new File(outDir.toString()); + dir.mkdir(); + files[i] = new File(dir,"testfile"+i); + FileWriter fileWriter = new FileWriter(files[i]); + fileWriter.close(); + paths[i] = new Path(outDir+"/testfile"+i); + } + CombineFileSplit combineFileSplit = new CombineFileSplit(conf, paths, fileLength); + Reporter reporter = Mockito.mock(Reporter.class); + CombineFileRecordReader cfrr = new CombineFileRecordReader(conf, combineFileSplit, + reporter, TextRecordReaderWrapper.class); + verify(reporter).progress(); + Assert.assertFalse(cfrr.next(key,value)); + verify(reporter, times(3)).progress(); + } finally { + FileUtil.fullyDelete(new File(outDir.toString())); + } + + } +} + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileRecordReader.java new file mode 100644 index 00000000000..052195efcdc --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileRecordReader.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce.lib.input; + +import java.io.FileWriter; +import java.io.IOException; +import java.io.File; + +import junit.framework.Assert; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; +import org.apache.hadoop.mapreduce.TaskAttemptID; +import org.apache.hadoop.mapred.Task.TaskReporter; + +import org.mockito.Mockito; + +import org.junit.Test; + +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class TestCombineFileRecordReader { + + private static Path outDir = new Path(System.getProperty("test.build.data", + "/tmp"), TestCombineFileRecordReader.class.getName()); + private static class TextRecordReaderWrapper + extends CombineFileRecordReaderWrapper { + // this constructor signature is required by CombineFileRecordReader + public TextRecordReaderWrapper(org.apache.hadoop.mapreduce.lib.input.CombineFileSplit split, + TaskAttemptContext context, Integer idx) + throws IOException, InterruptedException { + super(new TextInputFormat(), split, context, idx); + } + } + + @SuppressWarnings("unchecked") + @Test + public void testProgressIsReportedIfInputASeriesOfEmptyFiles() throws IOException, InterruptedException { + JobConf conf = new JobConf(); + Path[] paths = new Path[3]; + File[] files = new File[3]; + long[] fileLength = new long[3]; + + try { + for(int i=0;i<3;i++){ + File dir = new File(outDir.toString()); + dir.mkdir(); + files[i] = new File(dir,"testfile"+i); + FileWriter fileWriter = new FileWriter(files[i]); + fileWriter.flush(); + fileWriter.close(); + fileLength[i] = i; + paths[i] = new Path(outDir+"/testfile"+i); + } + + CombineFileSplit combineFileSplit = new CombineFileSplit(paths, fileLength); + TaskAttemptID taskAttemptID = Mockito.mock(TaskAttemptID.class); + TaskReporter reporter = Mockito.mock(TaskReporter.class); + TaskAttemptContextImpl taskAttemptContext = + new TaskAttemptContextImpl(conf, taskAttemptID,reporter); + + CombineFileRecordReader cfrr = new CombineFileRecordReader(combineFileSplit, + taskAttemptContext, TextRecordReaderWrapper.class); + + cfrr.initialize(combineFileSplit,taskAttemptContext); + + verify(reporter).progress(); + Assert.assertFalse(cfrr.nextKeyValue()); + verify(reporter, times(3)).progress(); + } finally { + FileUtil.fullyDelete(new File(outDir.toString())); + } + } +} + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java index 382c1971f6c..3d704ef0530 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java @@ -40,6 +40,8 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapreduce.Counters; +import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.TaskID; import org.apache.hadoop.mapreduce.TypeConverter; @@ -51,7 +53,9 @@ import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.AMInfo; import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo; import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskAttemptInfo; import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo; +import org.apache.hadoop.mapreduce.jobhistory.TaskFailedEvent; import org.apache.hadoop.mapreduce.jobhistory.TaskFinishedEvent; +import org.apache.hadoop.mapreduce.jobhistory.TaskStartedEvent; import org.apache.hadoop.mapreduce.v2.api.records.JobId; import org.apache.hadoop.mapreduce.v2.api.records.JobState; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId; @@ -69,7 +73,6 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType; import org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo; import org.apache.hadoop.mapreduce.v2.hs.TestJobHistoryEvents.MRAppWithHistory; import org.apache.hadoop.mapreduce.v2.hs.webapp.dao.JobsInfo; -import org.apache.hadoop.mapreduce.v2.jobhistory.FileNameIndexUtils; import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils; import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo; import org.apache.hadoop.net.DNSToSwitchMapping; @@ -730,4 +733,40 @@ public class TestJobHistoryParsing { assertNull(test.getAMInfos()); } + + @Test + public void testMultipleFailedTasks() throws Exception { + JobHistoryParser parser = + new JobHistoryParser(Mockito.mock(FSDataInputStream.class)); + EventReader reader = Mockito.mock(EventReader.class); + final AtomicInteger numEventsRead = new AtomicInteger(0); // Hack! + final org.apache.hadoop.mapreduce.TaskType taskType = + org.apache.hadoop.mapreduce.TaskType.MAP; + final TaskID[] tids = new TaskID[2]; + JobID jid = new JobID("1", 1); + tids[0] = new TaskID(jid, taskType, 0); + tids[1] = new TaskID(jid, taskType, 1); + Mockito.when(reader.getNextEvent()).thenAnswer( + new Answer() { + public HistoryEvent answer(InvocationOnMock invocation) + throws IOException { + // send two task start and two task fail events for tasks 0 and 1 + int eventId = numEventsRead.getAndIncrement(); + TaskID tid = tids[eventId & 0x1]; + if (eventId < 2) { + return new TaskStartedEvent(tid, 0, taskType, ""); + } + if (eventId < 4) { + TaskFailedEvent tfe = new TaskFailedEvent(tid, 0, taskType, + "failed", "FAILED", null, new Counters()); + tfe.setDatum(tfe.getDatum()); + return tfe; + } + return null; + } + }); + JobInfo info = parser.parse(reader); + assertTrue("Task 0 not implicated", + info.getErrorInfo().contains(tids[0].toString())); + } } diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 3039c6f9a07..8b880d33f91 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -6,20 +6,6 @@ Trunk - Unreleased NEW FEATURES - YARN-1496. Protocol additions to allow moving apps between queues (Sandy - Ryza) - - YARN-1498. Common scheduler changes for moving apps between queues (Sandy - Ryza) - - YARN-1504. RM changes for moving apps between queues (Sandy Ryza) - - YARN-1499. Fair Scheduler changes for moving apps between queues (Sandy - Ryza) - - YARN-1497. Command line additions for moving apps between queues (Sandy - Ryza) - IMPROVEMENTS OPTIMIZATIONS @@ -119,6 +105,23 @@ Release 2.4.0 - UNRELEASED YARN-1635. Implemented a Leveldb based ApplicationTimelineStore. (Billie Rinaldi via zjshen) + YARN-1637. Implemented a client library for Java users to post timeline + entities and events. (zjshen) + + YARN-1496. Protocol additions to allow moving apps between queues (Sandy + Ryza) + + YARN-1498. Common scheduler changes for moving apps between queues (Sandy + Ryza) + + YARN-1504. RM changes for moving apps between queues (Sandy Ryza) + + YARN-1499. Fair Scheduler changes for moving apps between queues (Sandy + Ryza) + + YARN-1497. Command line additions for moving apps between queues (Sandy + Ryza) + IMPROVEMENTS YARN-1007. Enhance History Reader interface for Containers. (Mayank Bansal via @@ -166,6 +169,25 @@ Release 2.4.0 - UNRELEASED YARN-1493. Changed ResourceManager and Scheduler interfacing to recognize app-attempts separately from apps. (Jian He via vinodkv) + YARN-1459. Changed ResourceManager to depend its service initialization + on the configuration-provider mechanism during startup too. (Xuan Gong via + vinodkv) + + YARN-1706. Created an utility method to dump timeline records to JSON + strings. (zjshen) + + YARN-1641. ZK store should attempt a write periodically to ensure it is + still Active. (kasha) + + YARN-1531. True up yarn command documentation (Akira Ajisaka via kasha) + + YARN-1345. Remove FINAL_SAVING state from YarnApplicationAttemptState + (Zhijie Shen via jianhe) + + YARN-1676. Modified RM HA handling of user-to-group mappings to + be available across RM failover by making using of a remote + configuration-provider. (Xuan Gong via vinodkv) + OPTIMIZATIONS BUG FIXES @@ -233,7 +255,37 @@ Release 2.4.0 - UNRELEASED YARN-1672. YarnConfiguration is missing a default for yarn.nodemanager.log.retain-seconds (Naren Koneru via kasha) -Release 2.3.0 - UNRELEASED + YARN-1698. Fixed default TimelineStore in code to match what is documented + in yarn-default.xml (Zhijie Shen via vinodkv) + + YARN-1697. NodeManager reports negative running containers (Sandy Ryza) + + YARN-1719. Fixed the root path related Jersey warnings produced in + ATSWebServices. (Billie Rinaldi via zjshen) + + YARN-1692. ConcurrentModificationException in fair scheduler AppSchedulable + (Sangjin Lee via Sandy Ryza) + + YARN-1578. Fixed reading incomplete application attempt and container data + in FileSystemApplicationHistoryStore. (Shinichi Yamashita via zjshen) + + YARN-1417. Modified RM to generate container-tokens not at creation time, but + at allocation time so as to prevent RM from shelling out containers with + expired tokens. (Omkar Vinit Joshi and Jian He via vinodkv) + +Release 2.3.1 - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + +Release 2.3.0 - 2014-02-18 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index 74ca61b8578..0fac0b98f1f 100644 --- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -309,4 +309,10 @@ + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationAttemptState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationAttemptState.java index 8b180a1f8ea..1ae9f9e2726 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationAttemptState.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationAttemptState.java @@ -51,9 +51,6 @@ public enum YarnApplicationAttemptState { /** AppAttempt is currently running. */ RUNNING, - /** AppAttempt is waiting for state bing saved */ - FINAL_SAVING, - /** AppAttempt is finishing. */ FINISHING, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java index 78c34d9de98..b31573d39eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.conf; import java.io.IOException; - import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; @@ -34,8 +33,8 @@ import org.apache.hadoop.yarn.exceptions.YarnException; */ public abstract class ConfigurationProvider { - public void init(Configuration conf) throws Exception { - initInternal(conf); + public void init(Configuration bootstrapConf) throws Exception { + initInternal(bootstrapConf); } public void close() throws Exception { @@ -43,19 +42,21 @@ public abstract class ConfigurationProvider { } /** - * Get the configuration. + * Get the configuration and combine with bootstrapConf + * @param bootstrapConf Configuration * @param name The configuration file name * @return configuration * @throws YarnException * @throws IOException */ - public abstract Configuration getConfiguration(String name) - throws YarnException, IOException; + public abstract Configuration getConfiguration(Configuration bootstrapConf, + String name) throws YarnException, IOException; /** * Derived classes initialize themselves using this method. */ - public abstract void initInternal(Configuration conf) throws Exception; + public abstract void initInternal(Configuration bootstrapConf) + throws Exception; /** * Derived classes close themselves using this method. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java index 4adc72e1f11..3562f173acb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java @@ -33,12 +33,12 @@ public class ConfigurationProviderFactory { /** * Creates an instance of {@link ConfigurationProvider} using given * configuration. - * @param conf + * @param bootstrapConf * @return configurationProvider */ @SuppressWarnings("unchecked") public static ConfigurationProvider - getConfigurationProvider(Configuration conf) { + getConfigurationProvider(Configuration bootstrapConf) { Class defaultProviderClass; try { defaultProviderClass = (Class) @@ -49,9 +49,11 @@ public class ConfigurationProviderFactory { "Invalid default configuration provider class" + YarnConfiguration.DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS, e); } - ConfigurationProvider configurationProvider = ReflectionUtils.newInstance( - conf.getClass(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, - defaultProviderClass, ConfigurationProvider.class), conf); + ConfigurationProvider configurationProvider = + ReflectionUtils.newInstance(bootstrapConf.getClass( + YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + defaultProviderClass, ConfigurationProvider.class), + bootstrapConf); return configurationProvider; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 8f6cf4c783b..48aac9d08e5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -120,10 +120,9 @@ enum YarnApplicationAttemptStateProto { APP_ATTEMPT_LAUNCHED = 6; APP_ATTEMPT_FAILED = 7; APP_ATTEMPT_RUNNING = 8; - APP_ATTEMPT_FINAL_SAVING = 9; - APP_ATTEMPT_FINISHING = 10; - APP_ATTEMPT_FINISHED = 11; - APP_ATTEMPT_KILLED = 12; + APP_ATTEMPT_FINISHING = 9; + APP_ATTEMPT_FINISHED = 10; + APP_ATTEMPT_KILLED = 11; } enum FinalApplicationStatusProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml index 54da659fee6..6091686a036 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml @@ -79,6 +79,10 @@ org.mortbay.jetty jetty-util + + com.sun.jersey + jersey-client + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java new file mode 100644 index 00000000000..8be00ac6ff6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client.api; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntity; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors; +import org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl; +import org.apache.hadoop.yarn.exceptions.YarnException; + +/** + * A client library that can be used to post some information in terms of a + * number of conceptual entities. + * + * @See ATSEntity + */ +@Public +@Unstable +public abstract class TimelineClient extends AbstractService { + + @Public + public static TimelineClient createTimelineClient() { + TimelineClient client = new TimelineClientImpl(); + return client; + } + + @Private + protected TimelineClient(String name) { + super(name); + } + + /** + *

+ * Post the information of a number of conceptual entities of an application + * to the timeline server. It is a blocking API. The method will not return + * until it gets the response from the timeline server. + *

+ * + * @param entities + * the collection of {@link ATSEntity} + * @return the error information if the post entities are not correctly stored + * @throws IOException + * @throws YarnException + */ + @Public + public abstract ATSPutErrors postEntities( + ATSEntity... entities) throws IOException, YarnException; + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java new file mode 100644 index 00000000000..9fcc2bd6e3d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client.api.impl; + +import java.io.IOException; +import java.net.URI; +import java.util.Arrays; + +import javax.ws.rs.core.MediaType; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.HttpConfig; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntity; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors; +import org.apache.hadoop.yarn.client.api.TimelineClient; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.api.client.config.ClientConfig; +import com.sun.jersey.api.client.config.DefaultClientConfig; + +@Private +@Unstable +public class TimelineClientImpl extends TimelineClient { + + private static final Log LOG = LogFactory.getLog(TimelineClientImpl.class); + private static final String RESOURCE_URI_STR = "/ws/v1/apptimeline/"; + private static final Joiner JOINER = Joiner.on(""); + + private Client client; + private URI resURI; + + public TimelineClientImpl() { + super(TimelineClientImpl.class.getName()); + ClientConfig cc = new DefaultClientConfig(); + cc.getClasses().add(YarnJacksonJaxbJsonProvider.class); + client = Client.create(cc); + } + + protected void serviceInit(Configuration conf) throws Exception { + resURI = new URI(JOINER.join(HttpConfig.getSchemePrefix(), + HttpConfig.isSecure() ? conf.get( + YarnConfiguration.AHS_WEBAPP_HTTPS_ADDRESS, + YarnConfiguration.DEFAULT_AHS_WEBAPP_HTTPS_ADDRESS) : conf.get( + YarnConfiguration.AHS_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_AHS_WEBAPP_ADDRESS), RESOURCE_URI_STR)); + super.serviceInit(conf); + } + + @Override + public ATSPutErrors postEntities( + ATSEntity... entities) throws IOException, YarnException { + ATSEntities entitiesContainer = new ATSEntities(); + entitiesContainer.addEntities(Arrays.asList(entities)); + ClientResponse resp = doPostingEntities(entitiesContainer); + if (resp.getClientResponseStatus() != ClientResponse.Status.OK) { + String msg = + "Failed to get the response from the timeline server."; + LOG.error(msg); + if (LOG.isDebugEnabled()) { + String output = resp.getEntity(String.class); + LOG.debug("HTTP error code: " + resp.getStatus() + + " Server response : \n" + output); + } + throw new YarnException(msg); + } + return resp.getEntity(ATSPutErrors.class); + } + + @Private + @VisibleForTesting + public ClientResponse doPostingEntities(ATSEntities entities) { + WebResource webResource = client.resource(resURI); + return webResource.accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON) + .post(ClientResponse.class, entities); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 80e548d26e6..4332f5beeaf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -382,11 +382,7 @@ public class ApplicationCLI extends YarnCLI { } /** - * Kills the application with the application id as appId - * - * @param applicationId - * @throws YarnException - * @throws IOException + * Moves the application with the given ID to the given queue. */ private void moveApplicationAcrossQueues(String applicationId, String queue) throws YarnException, IOException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java new file mode 100644 index 00000000000..a3917a2da57 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java @@ -0,0 +1,137 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client.api.impl; + +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; +import junit.framework.Assert; + +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntity; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEvent; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors; +import org.apache.hadoop.yarn.client.api.TimelineClient; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.sun.jersey.api.client.ClientResponse; + +public class TestTimelineClient { + + private TimelineClientImpl client; + + @Before + public void setup() { + client = spy((TimelineClientImpl) TimelineClient.createTimelineClient()); + client.init(new YarnConfiguration()); + client.start(); + } + + @After + public void tearDown() { + client.stop(); + } + + @Test + public void testPostEntities() throws Exception { + mockClientResponse(ClientResponse.Status.OK, false); + try { + ATSPutErrors errors = client.postEntities(generateATSEntity()); + Assert.assertEquals(0, errors.getErrors().size()); + } catch (YarnException e) { + Assert.fail("Exception is not expected"); + } + } + + @Test + public void testPostEntitiesWithError() throws Exception { + mockClientResponse(ClientResponse.Status.OK, true); + try { + ATSPutErrors errors = client.postEntities(generateATSEntity()); + Assert.assertEquals(1, errors.getErrors().size()); + Assert.assertEquals("test entity id", errors.getErrors().get(0) + .getEntityId()); + Assert.assertEquals("test entity type", errors.getErrors().get(0) + .getEntityType()); + Assert.assertEquals(ATSPutErrors.ATSPutError.IO_EXCEPTION, + errors.getErrors().get(0).getErrorCode()); + } catch (YarnException e) { + Assert.fail("Exception is not expected"); + } + } + + @Test + public void testPostEntitiesNoResponse() throws Exception { + mockClientResponse(ClientResponse.Status.INTERNAL_SERVER_ERROR, false); + try { + client.postEntities(generateATSEntity()); + Assert.fail("Exception is expected"); + } catch (YarnException e) { + Assert.assertTrue(e.getMessage().contains( + "Failed to get the response from the timeline server.")); + } + } + + private ClientResponse mockClientResponse(ClientResponse.Status status, + boolean hasError) { + ClientResponse response = mock(ClientResponse.class); + doReturn(response).when(client) + .doPostingEntities(any(ATSEntities.class)); + when(response.getClientResponseStatus()).thenReturn(status); + ATSPutErrors.ATSPutError error = new ATSPutErrors.ATSPutError(); + error.setEntityId("test entity id"); + error.setEntityType("test entity type"); + error.setErrorCode(ATSPutErrors.ATSPutError.IO_EXCEPTION); + ATSPutErrors errors = new ATSPutErrors(); + if (hasError) { + errors.addError(error); + } + when(response.getEntity(ATSPutErrors.class)).thenReturn(errors); + return response; + } + + private static ATSEntity generateATSEntity() { + ATSEntity entity = new ATSEntity(); + entity.setEntityId("entity id"); + entity.setEntityType("entity type"); + entity.setStartTime(System.currentTimeMillis()); + for (int i = 0; i < 2; ++i) { + ATSEvent event = new ATSEvent(); + event.setTimestamp(System.currentTimeMillis()); + event.setEventType("test event type " + i); + event.addEventInfo("key1", "val1"); + event.addEventInfo("key2", "val2"); + entity.addEvent(event); + } + entity.addRelatedEntity("test ref type 1", "test ref id 1"); + entity.addRelatedEntity("test ref type 2", "test ref id 2"); + entity.addPrimaryFilter("pkey1", "pval1"); + entity.addPrimaryFilter("pkey2", "pval2"); + entity.addOtherInfo("okey1", "oval1"); + entity.addOtherInfo("okey2", "oval2"); + return entity; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 12bc6be7316..97721864968 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -675,7 +675,6 @@ public class TestYarnCLI { int result = spyCli.run(new String[] { "-help" }); Assert.assertTrue(result == 0); verify(spyCli).printUsage(any(Options.class)); - System.err.println(sysOutStream.toString()); //todo sandyt remove this hejfkdsl Assert.assertEquals(createApplicationCLIHelpMessage(), sysOutStream.toString()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java index 709f54a3529..390aace7d21 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn; import java.io.IOException; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -42,24 +41,24 @@ public class FileSystemBasedConfigurationProvider private Path configDir; @Override - public synchronized Configuration getConfiguration(String name) - throws IOException, YarnException { + public synchronized Configuration getConfiguration(Configuration bootstrapConf, + String name) throws IOException, YarnException { Path configPath = new Path(this.configDir, name); if (!fs.exists(configPath)) { throw new YarnException("Can not find Configuration: " + name + " in " + configDir); } - Configuration conf = new Configuration(false); - conf.addResource(fs.open(configPath)); - return conf; + bootstrapConf.addResource(fs.open(configPath)); + return bootstrapConf; } @Override - public synchronized void initInternal(Configuration conf) throws Exception { + public synchronized void initInternal(Configuration bootstrapConf) + throws Exception { configDir = - new Path(conf.get(YarnConfiguration.FS_BASED_RM_CONF_STORE, + new Path(bootstrapConf.get(YarnConfiguration.FS_BASED_RM_CONF_STORE, YarnConfiguration.DEFAULT_FS_BASED_RM_CONF_STORE)); - fs = configDir.getFileSystem(conf); + fs = configDir.getFileSystem(bootstrapConf); if (!fs.exists(configDir)) { fs.mkdirs(configDir); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java index d152c353f08..3e6996036f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn; import java.io.IOException; - import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; @@ -31,13 +30,13 @@ import org.apache.hadoop.yarn.exceptions.YarnException; public class LocalConfigurationProvider extends ConfigurationProvider { @Override - public Configuration getConfiguration(String name) - throws IOException, YarnException { - return new Configuration(); + public Configuration getConfiguration(Configuration bootstrapConf, + String name) throws IOException, YarnException { + return bootstrapConf; } @Override - public void initInternal(Configuration conf) throws Exception { + public void initInternal(Configuration bootstrapConf) throws Exception { // Do nothing } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/TimelineUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/TimelineUtils.java new file mode 100644 index 00000000000..4ab557e33e1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/TimelineUtils.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.util; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.codehaus.jackson.JsonGenerationException; +import org.codehaus.jackson.map.AnnotationIntrospector; +import org.codehaus.jackson.map.JsonMappingException; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.annotate.JsonSerialize.Inclusion; +import org.codehaus.jackson.xc.JaxbAnnotationIntrospector; + +/** + * The helper class for the timeline module. + * + */ +@Public +@Evolving +public class TimelineUtils { + + private static ObjectMapper mapper; + + static { + mapper = new ObjectMapper(); + AnnotationIntrospector introspector = new JaxbAnnotationIntrospector(); + mapper.setAnnotationIntrospector(introspector); + mapper.getSerializationConfig() + .setSerializationInclusion(Inclusion.NON_NULL); + } + + /** + * Serialize a POJO object into a JSON string not in a pretty format + * + * @param o + * an object to serialize + * @return a JSON string + * @throws IOException + * @throws JsonMappingException + * @throws JsonGenerationException + */ + public static String dumpTimelineRecordtoJSON(Object o) + throws JsonGenerationException, JsonMappingException, IOException { + return dumpTimelineRecordtoJSON(o, false); + } + + /** + * Serialize a POJO object into a JSON string + * + * @param o + * an object to serialize + * @param pretty + * whether in a pretty format or not + * @return a JSON string + * @throws IOException + * @throws JsonMappingException + * @throws JsonGenerationException + */ + public static String dumpTimelineRecordtoJSON(Object o, boolean pretty) + throws JsonGenerationException, JsonMappingException, IOException { + if (pretty) { + return mapper.defaultPrettyPrintingWriter().writeValueAsString(o); + } else { + return mapper.writeValueAsString(o); + } + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java index 24d1ce91e62..330e099364e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java @@ -19,18 +19,23 @@ package org.apache.hadoop.yarn.api.records.apptimeline; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import junit.framework.Assert; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors.ATSPutError; +import org.apache.hadoop.yarn.util.TimelineUtils; import org.junit.Test; public class TestApplicationTimelineRecords { + private static final Log LOG = + LogFactory.getLog(TestApplicationTimelineRecords.class); + @Test - public void testATSEntities() { + public void testATSEntities() throws Exception { ATSEntities entities = new ATSEntities(); for (int j = 0; j < 2; ++j) { ATSEntity entity = new ATSEntity(); @@ -53,6 +58,9 @@ public class TestApplicationTimelineRecords { entity.addOtherInfo("okey2", "oval2"); entities.addEntity(entity); } + LOG.info("Entities in JSON:"); + LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(entities, true)); + Assert.assertEquals(2, entities.getEntities().size()); ATSEntity entity1 = entities.getEntities().get(0); Assert.assertEquals("entity id 0", entity1.getEntityId()); @@ -71,7 +79,7 @@ public class TestApplicationTimelineRecords { } @Test - public void testATSEvents() { + public void testATSEvents() throws Exception { ATSEvents events = new ATSEvents(); for (int j = 0; j < 2; ++j) { ATSEvents.ATSEventsOfOneEntity partEvents = @@ -88,6 +96,9 @@ public class TestApplicationTimelineRecords { } events.addEvent(partEvents); } + LOG.info("Events in JSON:"); + LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(events, true)); + Assert.assertEquals(2, events.getAllEvents().size()); ATSEvents.ATSEventsOfOneEntity partEvents1 = events.getAllEvents().get(0); Assert.assertEquals("entity id 0", partEvents1.getEntityId()); @@ -112,7 +123,7 @@ public class TestApplicationTimelineRecords { } @Test - public void testATSPutErrors() { + public void testATSPutErrors() throws Exception { ATSPutErrors atsPutErrors = new ATSPutErrors(); ATSPutError error1 = new ATSPutError(); error1.setEntityId("entity id 1"); @@ -127,6 +138,8 @@ public class TestApplicationTimelineRecords { error2.setErrorCode(ATSPutError.IO_EXCEPTION); errors.add(error2); atsPutErrors.addErrors(errors); + LOG.info("Errors in JSON:"); + LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(atsPutErrors, true)); Assert.assertEquals(3, atsPutErrors.getErrors().size()); ATSPutError e = atsPutErrors.getErrors().get(0); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java index 4ec986065b6..73a09417a01 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java @@ -34,7 +34,7 @@ import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.ApplicationTimelineStore; -import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.MemoryApplicationTimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.LeveldbApplicationTimelineStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebApps; @@ -143,10 +143,8 @@ public class ApplicationHistoryServer extends CompositeService { protected ApplicationTimelineStore createApplicationTimelineStore( Configuration conf) { - // TODO: need to replace the MemoryApplicationTimelineStore.class with the - // LevelDB implementation return ReflectionUtils.newInstance(conf.getClass( - YarnConfiguration.ATS_STORE, MemoryApplicationTimelineStore.class, + YarnConfiguration.ATS_STORE, LeveldbApplicationTimelineStore.class, ApplicationTimelineStore.class), conf); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java index 9109dfccb15..a321976c5b1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java @@ -215,17 +215,30 @@ public class FileSystemApplicationHistoryStore extends AbstractService getApplicationAttempts(ApplicationId appId) throws IOException { Map historyDataMap = new HashMap(); - Map> startFinshDataMap = - new HashMap>(); HistoryFileReader hfReader = getHistoryFileReader(appId); try { while (hfReader.hasNext()) { HistoryFileReader.Entry entry = hfReader.next(); - if (entry.key.id.startsWith(ConverterUtils.APPLICATION_ATTEMPT_PREFIX)) { - if (entry.key.suffix.equals(START_DATA_SUFFIX)) { - retrieveStartFinishData(appId, entry, startFinshDataMap, true); - } else if (entry.key.suffix.equals(FINISH_DATA_SUFFIX)) { - retrieveStartFinishData(appId, entry, startFinshDataMap, false); + if (entry.key.id.startsWith( + ConverterUtils.APPLICATION_ATTEMPT_PREFIX)) { + ApplicationAttemptId appAttemptId = + ConverterUtils.toApplicationAttemptId(entry.key.id); + if (appAttemptId.getApplicationId().equals(appId)) { + ApplicationAttemptHistoryData historyData = + historyDataMap.get(appAttemptId); + if (historyData == null) { + historyData = ApplicationAttemptHistoryData.newInstance( + appAttemptId, null, -1, null, null, null, + FinalApplicationStatus.UNDEFINED, null); + historyDataMap.put(appAttemptId, historyData); + } + if (entry.key.suffix.equals(START_DATA_SUFFIX)) { + mergeApplicationAttemptHistoryData(historyData, + parseApplicationAttemptStartData(entry.value)); + } else if (entry.key.suffix.equals(FINISH_DATA_SUFFIX)) { + mergeApplicationAttemptHistoryData(historyData, + parseApplicationAttemptFinishData(entry.value)); + } } } } @@ -237,45 +250,9 @@ public class FileSystemApplicationHistoryStore extends AbstractService } finally { hfReader.close(); } - for (Map.Entry> entry : startFinshDataMap - .entrySet()) { - ApplicationAttemptHistoryData historyData = - ApplicationAttemptHistoryData.newInstance(entry.getKey(), null, -1, - null, null, null, FinalApplicationStatus.UNDEFINED, null); - mergeApplicationAttemptHistoryData(historyData, - entry.getValue().startData); - mergeApplicationAttemptHistoryData(historyData, - entry.getValue().finishData); - historyDataMap.put(entry.getKey(), historyData); - } return historyDataMap; } - private - void - retrieveStartFinishData( - ApplicationId appId, - HistoryFileReader.Entry entry, - Map> startFinshDataMap, - boolean start) throws IOException { - ApplicationAttemptId appAttemptId = - ConverterUtils.toApplicationAttemptId(entry.key.id); - if (appAttemptId.getApplicationId().equals(appId)) { - StartFinishDataPair pair = - startFinshDataMap.get(appAttemptId); - if (pair == null) { - pair = - new StartFinishDataPair(); - startFinshDataMap.put(appAttemptId, pair); - } - if (start) { - pair.startData = parseApplicationAttemptStartData(entry.value); - } else { - pair.finishData = parseApplicationAttemptFinishData(entry.value); - } - } - } - @Override public ApplicationAttemptHistoryData getApplicationAttempt( ApplicationAttemptId appAttemptId) throws IOException { @@ -391,20 +368,30 @@ public class FileSystemApplicationHistoryStore extends AbstractService ApplicationAttemptId appAttemptId) throws IOException { Map historyDataMap = new HashMap(); - Map> startFinshDataMap = - new HashMap>(); HistoryFileReader hfReader = getHistoryFileReader(appAttemptId.getApplicationId()); try { while (hfReader.hasNext()) { HistoryFileReader.Entry entry = hfReader.next(); if (entry.key.id.startsWith(ConverterUtils.CONTAINER_PREFIX)) { - if (entry.key.suffix.equals(START_DATA_SUFFIX)) { - retrieveStartFinishData(appAttemptId, entry, startFinshDataMap, - true); - } else if (entry.key.suffix.equals(FINISH_DATA_SUFFIX)) { - retrieveStartFinishData(appAttemptId, entry, startFinshDataMap, - false); + ContainerId containerId = + ConverterUtils.toContainerId(entry.key.id); + if (containerId.getApplicationAttemptId().equals(appAttemptId)) { + ContainerHistoryData historyData = + historyDataMap.get(containerId); + if (historyData == null) { + historyData = ContainerHistoryData.newInstance( + containerId, null, null, null, Long.MIN_VALUE, + Long.MAX_VALUE, null, null, Integer.MAX_VALUE, null); + historyDataMap.put(containerId, historyData); + } + if (entry.key.suffix.equals(START_DATA_SUFFIX)) { + mergeContainerHistoryData(historyData, + parseContainerStartData(entry.value)); + } else if (entry.key.suffix.equals(FINISH_DATA_SUFFIX)) { + mergeContainerHistoryData(historyData, + parseContainerFinishData(entry.value)); + } } } } @@ -416,43 +403,9 @@ public class FileSystemApplicationHistoryStore extends AbstractService } finally { hfReader.close(); } - for (Map.Entry> entry : startFinshDataMap - .entrySet()) { - ContainerHistoryData historyData = - ContainerHistoryData - .newInstance(entry.getKey(), null, null, null, Long.MIN_VALUE, - Long.MAX_VALUE, null, null, Integer.MAX_VALUE, null); - mergeContainerHistoryData(historyData, entry.getValue().startData); - mergeContainerHistoryData(historyData, entry.getValue().finishData); - historyDataMap.put(entry.getKey(), historyData); - } return historyDataMap; } - private - void - retrieveStartFinishData( - ApplicationAttemptId appAttemptId, - HistoryFileReader.Entry entry, - Map> startFinshDataMap, - boolean start) throws IOException { - ContainerId containerId = ConverterUtils.toContainerId(entry.key.id); - if (containerId.getApplicationAttemptId().equals(appAttemptId)) { - StartFinishDataPair pair = - startFinshDataMap.get(containerId); - if (pair == null) { - pair = - new StartFinishDataPair(); - startFinshDataMap.put(containerId, pair); - } - if (start) { - pair.startData = parseContainerStartData(entry.value); - } else { - pair.finishData = parseContainerFinishData(entry.value); - } - } - } - @Override public void applicationStarted(ApplicationStartData appStart) throws IOException { @@ -828,14 +781,5 @@ public class FileSystemApplicationHistoryStore extends AbstractService id = in.readUTF(); suffix = in.readUTF(); } - } - - private static class StartFinishDataPair { - - private S startData; - private F finishData; - - } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java index 063b67afd07..baf00d68f16 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java @@ -107,7 +107,6 @@ public class ATSWebServices { * Return the description of the application timeline web services. */ @GET - @Path("/") @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */}) public AboutInfo about( @Context HttpServletRequest req, @@ -235,7 +234,6 @@ public class ATSWebServices { * that happen during storing. */ @POST - @Path("/") @Consumes({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */}) public ATSPutErrors postEntities( @Context HttpServletRequest req, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java index c31efab1bb6..960b68f3d43 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java @@ -72,6 +72,12 @@ public class TestFileSystemApplicationHistoryStore extends } private void testWriteHistoryData(int num) throws IOException { + testWriteHistoryData(num, false, false); + } + + private void testWriteHistoryData( + int num, boolean missingContainer, boolean missingApplicationAttempt) + throws IOException { // write application history data for (int i = 1; i <= num; ++i) { ApplicationId appId = ApplicationId.newInstance(0, i); @@ -83,21 +89,31 @@ public class TestFileSystemApplicationHistoryStore extends ApplicationAttemptId.newInstance(appId, j); writeApplicationAttemptStartData(appAttemptId); + if (missingApplicationAttempt && j == num) { + continue; + } // write container history data for (int k = 1; k <= num; ++k) { ContainerId containerId = ContainerId.newInstance(appAttemptId, k); writeContainerStartData(containerId); + if (missingContainer && k == num) { + continue; + } writeContainerFinishData(containerId); - - writeApplicationAttemptFinishData(appAttemptId); } + writeApplicationAttemptFinishData(appAttemptId); } - writeApplicationFinishData(appId); } } private void testReadHistoryData(int num) throws IOException { + testReadHistoryData(num, false, false); + } + + private void testReadHistoryData( + int num, boolean missingContainer, boolean missingApplicationAttempt) + throws IOException { // read application history data Assert.assertEquals(num, store.getAllApplications().size()); for (int i = 1; i <= num; ++i) { @@ -116,8 +132,14 @@ public class TestFileSystemApplicationHistoryStore extends store.getApplicationAttempt(appAttemptId); Assert.assertNotNull(attemptData); Assert.assertEquals(appAttemptId.toString(), attemptData.getHost()); - Assert.assertEquals(appAttemptId.toString(), - attemptData.getDiagnosticsInfo()); + + if (missingApplicationAttempt && j == num) { + Assert.assertNull(attemptData.getDiagnosticsInfo()); + continue; + } else { + Assert.assertEquals(appAttemptId.toString(), + attemptData.getDiagnosticsInfo()); + } // read container history data Assert.assertEquals(num, store.getContainers(appAttemptId).size()); @@ -127,8 +149,12 @@ public class TestFileSystemApplicationHistoryStore extends Assert.assertNotNull(containerData); Assert.assertEquals(Priority.newInstance(containerId.getId()), containerData.getPriority()); - Assert.assertEquals(containerId.toString(), - containerData.getDiagnosticsInfo()); + if (missingContainer && k == num) { + Assert.assertNull(containerData.getDiagnosticsInfo()); + } else { + Assert.assertEquals(containerId.toString(), + containerData.getDiagnosticsInfo()); + } } ContainerHistoryData masterContainer = store.getAMContainer(appAttemptId); @@ -193,4 +219,15 @@ public class TestFileSystemApplicationHistoryStore extends Assert.assertTrue((usedDiskAfter - usedDiskBefore) < 20); } + @Test + public void testMissingContainerHistoryData() throws IOException { + testWriteHistoryData(3, true, false); + testReadHistoryData(3, true, false); + } + + @Test + public void testMissingApplicationAttemptHistoryData() throws IOException { + testWriteHistoryData(3, false, true); + testReadHistoryData(3, false, true); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index 486f3ce00c5..862e3fa9bcd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -83,6 +83,7 @@ public class ContainerImpl implements Container { private final String user; private int exitCode = ContainerExitStatus.INVALID; private final StringBuilder diagnostics; + private boolean wasLaunched; /** The NM-wide configuration - not specific to this container */ private final Configuration daemonConf; @@ -418,7 +419,9 @@ public class ContainerImpl implements Container { applicationId, containerId); break; case EXITED_WITH_FAILURE: - metrics.endRunningContainer(); + if (wasLaunched) { + metrics.endRunningContainer(); + } // fall through case LOCALIZATION_FAILED: metrics.failedContainer(); @@ -428,7 +431,9 @@ public class ContainerImpl implements Container { applicationId, containerId); break; case CONTAINER_CLEANEDUP_AFTER_KILL: - metrics.endRunningContainer(); + if (wasLaunched) { + metrics.endRunningContainer(); + } // fall through case NEW: metrics.killedContainer(); @@ -636,6 +641,7 @@ public class ContainerImpl implements Container { new ContainerStartMonitoringEvent(container.containerId, vmemBytes, pmemBytes)); container.metrics.runningContainer(); + container.wasLaunched = true; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java index 4d62247539a..1feb8c70e77 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java @@ -99,4 +99,8 @@ public class NodeManagerMetrics { public void addResource(Resource res) { availableGB.incr(res.getMemory() / 1024); } + + public int getRunningContainers() { + return containersRunning.value(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java index addb28d83a2..3199fdfeeb6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java @@ -348,6 +348,9 @@ public class TestContainer { wc.c.getContainerState()); assertNull(wc.c.getLocalizedResources()); verifyCleanupCall(wc); + wc.c.handle(new ContainerEvent(wc.c.getContainerId(), + ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP)); + assertEquals(0, metrics.getRunningContainers()); } finally { if (wc != null) { wc.finished(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index d9c239e220a..6ebf90a6b5f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -26,6 +26,7 @@ import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.ha.HAServiceStatus; @@ -45,11 +46,8 @@ import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.service.CompositeService; -import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.ResourceOption; -import org.apache.hadoop.yarn.conf.ConfigurationProvider; -import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -92,8 +90,6 @@ public class AdminService extends CompositeService implements private Server server; private InetSocketAddress masterServiceAddress; private AccessControlList adminAcl; - - private ConfigurationProvider configurationProvider = null; private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); @@ -115,10 +111,6 @@ public class AdminService extends CompositeService implements } } - this.configurationProvider = - ConfigurationProviderFactory.getConfigurationProvider(conf); - configurationProvider.init(conf); - masterServiceAddress = conf.getSocketAddr( YarnConfiguration.RM_ADMIN_ADDRESS, YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS, @@ -139,9 +131,6 @@ public class AdminService extends CompositeService implements @Override protected synchronized void serviceStop() throws Exception { stopServer(); - if (this.configurationProvider != null) { - configurationProvider.close(); - } super.serviceStop(); } @@ -158,7 +147,10 @@ public class AdminService extends CompositeService implements if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { - refreshServiceAcls(conf, new RMPolicyProvider()); + refreshServiceAcls( + getConfiguration(conf, + YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE), + RMPolicyProvider.getInstance()); } if (rmContext.isHAEnabled()) { @@ -321,8 +313,8 @@ public class AdminService extends CompositeService implements RefreshQueuesResponse response = recordFactory.newRecordInstance(RefreshQueuesResponse.class); try { - Configuration conf = - getConfiguration(YarnConfiguration.CS_CONFIGURATION_FILE); + Configuration conf = getConfiguration(getConfig(), + YarnConfiguration.CS_CONFIGURATION_FILE); rmContext.getScheduler().reinitialize(conf, this.rmContext); RMAuditLogger.logSuccess(user.getShortUserName(), argName, "AdminService"); @@ -376,7 +368,8 @@ public class AdminService extends CompositeService implements } Configuration conf = - getConfiguration(YarnConfiguration.CORE_SITE_CONFIGURATION_FILE); + getConfiguration(getConfig(), + YarnConfiguration.CORE_SITE_CONFIGURATION_FILE); ProxyUsers.refreshSuperUserGroupsConfiguration(conf); RMAuditLogger.logSuccess(user.getShortUserName(), argName, "AdminService"); @@ -388,21 +381,22 @@ public class AdminService extends CompositeService implements @Override public RefreshUserToGroupsMappingsResponse refreshUserToGroupsMappings( RefreshUserToGroupsMappingsRequest request) - throws YarnException, StandbyException { - UserGroupInformation user = checkAcls("refreshUserToGroupsMappings"); + throws YarnException, IOException { + String argName = "refreshUserToGroupsMappings"; + UserGroupInformation user = checkAcls(argName); - // TODO (YARN-1459): Revisit handling user-groups on Standby RM if (!isRMActive()) { - RMAuditLogger.logFailure(user.getShortUserName(), - "refreshUserToGroupsMapping", + RMAuditLogger.logFailure(user.getShortUserName(), argName, adminAcl.toString(), "AdminService", "ResourceManager is not active. Can not refresh user-groups."); throwStandbyException(); } - Groups.getUserToGroupsMappingService().refresh(); - RMAuditLogger.logSuccess(user.getShortUserName(), - "refreshUserToGroupsMappings", "AdminService"); + Groups.getUserToGroupsMappingService( + getConfiguration(getConfig(), + YarnConfiguration.CORE_SITE_CONFIGURATION_FILE)).refresh(); + + RMAuditLogger.logSuccess(user.getShortUserName(), argName, "AdminService"); return recordFactory.newRecordInstance( RefreshUserToGroupsMappingsResponse.class); @@ -421,7 +415,7 @@ public class AdminService extends CompositeService implements throwStandbyException(); } Configuration conf = - getConfiguration(YarnConfiguration.YARN_SITE_XML_FILE); + getConfiguration(getConfig(), YarnConfiguration.YARN_SITE_XML_FILE); adminAcl = new AccessControlList(conf.get( YarnConfiguration.YARN_ADMIN_ACL, YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)); @@ -452,9 +446,10 @@ public class AdminService extends CompositeService implements throwStandbyException(); } - PolicyProvider policyProvider = new RMPolicyProvider(); + PolicyProvider policyProvider = RMPolicyProvider.getInstance(); Configuration conf = - getConfiguration(YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE); + getConfiguration(getConfig(), + YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE); refreshServiceAcls(conf, policyProvider); rmContext.getClientRMService().refreshServiceAcls(conf, policyProvider); @@ -466,12 +461,13 @@ public class AdminService extends CompositeService implements return recordFactory.newRecordInstance(RefreshServiceAclsResponse.class); } - synchronized void refreshServiceAcls(Configuration configuration, + private synchronized void refreshServiceAcls(Configuration configuration, PolicyProvider policyProvider) { - if (this.configurationProvider instanceof LocalConfigurationProvider) { + if (this.rmContext.getConfigurationProvider() instanceof + LocalConfigurationProvider) { this.server.refreshServiceAcl(configuration, policyProvider); } else { - this.server.refreshServiceAclWithConfigration(configuration, + this.server.refreshServiceAclWithLoadedConfiguration(configuration, policyProvider); } } @@ -521,9 +517,10 @@ public class AdminService extends CompositeService implements return response; } - private synchronized Configuration getConfiguration(String confFileName) - throws YarnException, IOException { - return this.configurationProvider.getConfiguration(confFileName); + private synchronized Configuration getConfiguration(Configuration conf, + String confFileName) throws YarnException, IOException { + return this.rmContext.getConfigurationProvider().getConfiguration(conf, + confFileName); } @VisibleForTesting diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index 2c4be13ee92..0c56134b811 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -105,7 +105,6 @@ public class ApplicationMasterService extends AbstractService implements private final AllocateResponse resync = recordFactory.newRecordInstance(AllocateResponse.class); private final RMContext rmContext; - private boolean useLocalConfigurationProvider; public ApplicationMasterService(RMContext rmContext, YarnScheduler scheduler) { super(ApplicationMasterService.class.getName()); @@ -115,15 +114,6 @@ public class ApplicationMasterService extends AbstractService implements this.rmContext = rmContext; } - @Override - protected void serviceInit(Configuration conf) throws Exception { - this.useLocalConfigurationProvider = - (LocalConfigurationProvider.class.isAssignableFrom(conf.getClass( - YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, - LocalConfigurationProvider.class))); - super.serviceInit(conf); - } - @Override protected void serviceStart() throws Exception { Configuration conf = getConfig(); @@ -150,7 +140,10 @@ public class ApplicationMasterService extends AbstractService implements if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { - refreshServiceAcls(conf, new RMPolicyProvider()); + refreshServiceAcls( + this.rmContext.getConfigurationProvider().getConfiguration(conf, + YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE), + RMPolicyProvider.getInstance()); } this.server.start(); @@ -591,10 +584,11 @@ public class ApplicationMasterService extends AbstractService implements public void refreshServiceAcls(Configuration configuration, PolicyProvider policyProvider) { - if (this.useLocalConfigurationProvider) { + if (this.rmContext.getConfigurationProvider() instanceof + LocalConfigurationProvider) { this.server.refreshServiceAcl(configuration, policyProvider); } else { - this.server.refreshServiceAclWithConfigration(configuration, + this.server.refreshServiceAclWithLoadedConfiguration(configuration, policyProvider); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 2f8526a7c71..43e94edd1a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -136,7 +136,6 @@ public class ClientRMService extends AbstractService implements private final ApplicationACLsManager applicationsACLsManager; private final QueueACLsManager queueACLsManager; - private boolean useLocalConfigurationProvider; public ClientRMService(RMContext rmContext, YarnScheduler scheduler, RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager, @@ -154,10 +153,6 @@ public class ClientRMService extends AbstractService implements @Override protected void serviceInit(Configuration conf) throws Exception { clientBindAddress = getBindAddress(conf); - this.useLocalConfigurationProvider = - (LocalConfigurationProvider.class.isAssignableFrom(conf.getClass( - YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, - LocalConfigurationProvider.class))); super.serviceInit(conf); } @@ -176,7 +171,10 @@ public class ClientRMService extends AbstractService implements if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { - refreshServiceAcls(conf, new RMPolicyProvider()); + refreshServiceAcls( + this.rmContext.getConfigurationProvider().getConfiguration(conf, + YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE), + RMPolicyProvider.getInstance()); } this.server.start(); @@ -809,10 +807,11 @@ public class ClientRMService extends AbstractService implements void refreshServiceAcls(Configuration configuration, PolicyProvider policyProvider) { - if (this.useLocalConfigurationProvider) { + if (this.rmContext.getConfigurationProvider() instanceof + LocalConfigurationProvider) { this.server.refreshServiceAcl(configuration, policyProvider); } else { - this.server.refreshServiceAclWithConfigration(configuration, + this.server.refreshServiceAclWithLoadedConfiguration(configuration, policyProvider); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java index 64a4165feb4..79fb5dfa23e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java @@ -23,6 +23,7 @@ import java.util.concurrent.ConcurrentMap; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.conf.ConfigurationProvider; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; @@ -97,4 +98,5 @@ public interface RMContext { void setRMApplicationHistoryWriter( RMApplicationHistoryWriter rmApplicationHistoryWriter); + ConfigurationProvider getConfigurationProvider(); } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java index 79e59831e9d..689a0914190 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java @@ -23,8 +23,10 @@ import java.util.concurrent.ConcurrentMap; import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.conf.ConfigurationProvider; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; @@ -78,7 +80,7 @@ public class RMContextImpl implements RMContext { private ResourceTrackerService resourceTrackerService; private ApplicationMasterService applicationMasterService; private RMApplicationHistoryWriter rmApplicationHistoryWriter; - + private ConfigurationProvider configurationProvider; /** * Default constructor. To be used in conjunction with setter methods for * individual fields. @@ -119,8 +121,11 @@ public class RMContextImpl implements RMContext { } catch (Exception e) { assert false; } + + ConfigurationProvider provider = new LocalConfigurationProvider(); + setConfigurationProvider(provider); } - + @Override public Dispatcher getDispatcher() { return this.rmDispatcher; @@ -334,4 +339,13 @@ public class RMContextImpl implements RMContext { this.rmApplicationHistoryWriter = rmApplicationHistoryWriter; } + @Override + public ConfigurationProvider getConfigurationProvider() { + return this.configurationProvider; + } + + public void setConfigurationProvider( + ConfigurationProvider configurationProvider) { + this.configurationProvider = configurationProvider; + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index 5556255352c..e884d29e303 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -214,8 +214,6 @@ public class RMServerUtils { return YarnApplicationAttemptState.RUNNING; case FINISHING: return YarnApplicationAttemptState.FINISHING; - case FINAL_SAVING: - return YarnApplicationAttemptState.FINAL_SAVING; case FINISHED: return YarnApplicationAttemptState.FINISHED; case KILLED: diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 8575cd57d65..5ef58a74d8e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -33,6 +33,7 @@ import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.http.HttpConfig.Policy; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.source.JvmMetrics; +import org.apache.hadoop.security.Groups; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.AbstractService; @@ -42,10 +43,13 @@ import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.conf.ConfigurationProvider; +import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; @@ -154,7 +158,7 @@ public class ResourceManager extends CompositeService implements Recoverable { private boolean recoveryEnabled; private String webAppAddress; - + private ConfigurationProvider configurationProvider = null; /** End of Active services */ private Configuration conf; @@ -182,6 +186,24 @@ public class ResourceManager extends CompositeService implements Recoverable { this.conf = conf; this.rmContext = new RMContextImpl(); + this.configurationProvider = + ConfigurationProviderFactory.getConfigurationProvider(conf); + this.configurationProvider.init(this.conf); + rmContext.setConfigurationProvider(configurationProvider); + if (!(this.configurationProvider instanceof LocalConfigurationProvider)) { + // load yarn-site.xml + this.conf = + this.configurationProvider.getConfiguration(this.conf, + YarnConfiguration.YARN_SITE_XML_FILE); + // load core-site.xml + this.conf = + this.configurationProvider.getConfiguration(this.conf, + YarnConfiguration.CORE_SITE_CONFIGURATION_FILE); + // Do refreshUserToGroupsMappings with loaded core-site.xml + Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(this.conf) + .refresh(); + } + // register the handlers for all AlwaysOn services using setupDispatcher(). rmDispatcher = setupDispatcher(); addIfService(rmDispatcher); @@ -884,6 +906,9 @@ public class ResourceManager extends CompositeService implements Recoverable { if (fetcher != null) { fetcher.stop(); } + if (configurationProvider != null) { + configurationProvider.close(); + } super.serviceStop(); transitionToStandby(false); rmContext.setHAServiceState(HAServiceState.STOPPING); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index 4f74179717f..8136c056129 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -95,7 +95,6 @@ public class ResourceTrackerService extends AbstractService implements private int minAllocMb; private int minAllocVcores; - private boolean useLocalConfigurationProvider; static { resync.setNodeAction(NodeAction.RESYNC); @@ -145,10 +144,6 @@ public class ResourceTrackerService extends AbstractService implements YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION, YarnConfiguration.DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION); - this.useLocalConfigurationProvider = - (LocalConfigurationProvider.class.isAssignableFrom(conf.getClass( - YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, - LocalConfigurationProvider.class))); super.serviceInit(conf); } @@ -169,7 +164,10 @@ public class ResourceTrackerService extends AbstractService implements if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { - refreshServiceAcls(conf, new RMPolicyProvider()); + refreshServiceAcls( + this.rmContext.getConfigurationProvider().getConfiguration(conf, + YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE), + RMPolicyProvider.getInstance()); } this.server.start(); @@ -423,10 +421,11 @@ public class ResourceTrackerService extends AbstractService implements void refreshServiceAcls(Configuration configuration, PolicyProvider policyProvider) { - if (this.useLocalConfigurationProvider) { + if (this.rmContext.getConfigurationProvider() instanceof + LocalConfigurationProvider) { this.server.refreshServiceAcl(configuration, policyProvider); } else { - this.server.refreshServiceAclWithConfigration(configuration, + this.server.refreshServiceAclWithLoadedConfiguration(configuration, policyProvider); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java index e603e9f8e66..05bfb3bb49e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java @@ -676,11 +676,11 @@ public abstract class RMStateStore extends AbstractService { @SuppressWarnings("unchecked") /** - * In {#handleStoreEvent}, this method is called to notify the - * ResourceManager that the store operation has failed. + * This method is called to notify the ResourceManager that the store + * operation has failed. * @param failureCause the exception due to which the operation failed */ - private void notifyStoreOperationFailed(Exception failureCause) { + protected void notifyStoreOperationFailed(Exception failureCause) { RMFatalEventType type; if (failureCause instanceof StoreFencedException) { type = RMFatalEventType.STATE_STORE_FENCED; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java index d8fdaae0fdd..eebeee791bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java @@ -137,6 +137,7 @@ public class ZKRMStateStore extends RMStateStore { private String fencingNodePath; private Op createFencingNodePathOp; private Op deleteFencingNodePathOp; + private Thread verifyActiveStatusThread; private String zkRootNodeUsername; private final String zkRootNodePassword = Long.toString(random.nextLong()); @@ -258,6 +259,8 @@ public class ZKRMStateStore extends RMStateStore { createRootDir(zkRootNodePath); if (HAUtil.isHAEnabled(getConfig())){ fence(); + verifyActiveStatusThread = new VerifyActiveStatusThread(); + verifyActiveStatusThread.start(); } createRootDir(rmAppRoot); createRootDir(rmDTSecretManagerRoot); @@ -350,6 +353,10 @@ public class ZKRMStateStore extends RMStateStore { @Override protected synchronized void closeInternal() throws Exception { + if (verifyActiveStatusThread != null) { + verifyActiveStatusThread.interrupt(); + verifyActiveStatusThread.join(1000); + } closeZkClients(); } @@ -856,6 +863,32 @@ public class ZKRMStateStore extends RMStateStore { }.runWithRetries(); } + /** + * Helper class that periodically attempts creating a znode to ensure that + * this RM continues to be the Active. + */ + private class VerifyActiveStatusThread extends Thread { + private List emptyOpList = new ArrayList(); + + VerifyActiveStatusThread() { + super(VerifyActiveStatusThread.class.getName()); + } + + public void run() { + try { + while (true) { + doMultiWithRetries(emptyOpList); + Thread.sleep(zkSessionTimeout); + } + } catch (InterruptedException ie) { + LOG.info(VerifyActiveStatusThread.class.getName() + " thread " + + "interrupted! Exiting!"); + } catch (Exception e) { + notifyStoreOperationFailed(new StoreFencedException()); + } + } + } + private abstract class ZKAction { // run() expects synchronization on ZKRMStateStore.this abstract T run() throws KeeperException, InterruptedException; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index ce246db7ba3..4ca8c28243a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -157,6 +157,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { private RMAppAttemptEvent eventCausingFinalSaving; private RMAppAttemptState targetedFinalState; private RMAppAttemptState recoveredFinalState; + private RMAppAttemptState stateBeforeFinalSaving; private Object transitionTodo; private static final StateMachineFactoryContainerToken, only in secure-mode - */ - Token createContainerToken( - FiCaSchedulerApp application, Container container) { - return containerTokenSecretManager.createContainerToken( - container.getId(), container.getNodeId(), - application.getUser(), container.getResource()); - } - private Resource assignContainer(Resource clusterResource, FiCaSchedulerNode node, FiCaSchedulerApp application, Priority priority, ResourceRequest request, NodeType type, RMContainer rmContainer) { @@ -1345,14 +1335,6 @@ public class LeafQueue implements CSQueue { unreserve(application, priority, node, rmContainer); } - Token containerToken = - createContainerToken(application, container); - if (containerToken == null) { - // Something went wrong... - return Resources.none(); - } - container.setContainerToken(containerToken); - // Inform the application RMContainer allocatedContainer = application.allocate(type, node, priority, request, container); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index 9c34f2f5995..4be6b941d12 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -192,7 +192,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt { return Math.min(((float)requiredResources / clusterNodes), 1.0f); } - public Resource getTotalPendingRequests() { + public synchronized Resource getTotalPendingRequests() { Resource ret = Resource.newInstance(0, 0); for (ResourceRequest rr : appSchedulingInfo.getAllResourceRequests()) { // to avoid double counting we count only "ANY" resource requests diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java index 275061a5fa1..146994b4f7e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java @@ -82,10 +82,12 @@ public class AppSchedulable extends Schedulable { Resources.addTo(demand, app.getCurrentConsumption()); // Add up outstanding resource requests - for (Priority p : app.getPriorities()) { - for (ResourceRequest r : app.getResourceRequests(p).values()) { - Resource total = Resources.multiply(r.getCapability(), r.getNumContainers()); - Resources.addTo(demand, total); + synchronized (app) { + for (Priority p : app.getPriorities()) { + for (ResourceRequest r : app.getResourceRequests(p).values()) { + Resource total = Resources.multiply(r.getCapability(), r.getNumContainers()); + Resources.addTo(demand, total); + } } } } @@ -149,17 +151,11 @@ public class AppSchedulable extends Schedulable { NodeId nodeId = node.getRMNode().getNodeID(); ContainerId containerId = BuilderUtils.newContainerId(application .getApplicationAttemptId(), application.getNewContainerId()); - org.apache.hadoop.yarn.api.records.Token containerToken = - containerTokenSecretManager.createContainerToken(containerId, nodeId, - application.getUser(), capability); - if (containerToken == null) { - return null; // Try again later. - } // Create the container Container container = BuilderUtils.newContainer(containerId, nodeId, node.getRMNode() - .getHttpAddress(), capability, priority, containerToken); + .getHttpAddress(), capability, priority, null); return container; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index e33348a10d4..a2e01345abe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -654,20 +654,11 @@ public class FifoScheduler extends AbstractYarnScheduler implements NodeId nodeId = node.getRMNode().getNodeID(); ContainerId containerId = BuilderUtils.newContainerId(application .getApplicationAttemptId(), application.getNewContainerId()); - Token containerToken = null; - - containerToken = - this.rmContext.getContainerTokenSecretManager() - .createContainerToken(containerId, nodeId, application.getUser(), - capability); - if (containerToken == null) { - return i; // Try again later. - } // Create the container Container container = BuilderUtils.newContainer(containerId, nodeId, node.getRMNode() - .getHttpAddress(), capability, priority, containerToken); + .getHttpAddress(), capability, priority, null); // Allocate! diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/authorize/RMPolicyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/authorize/RMPolicyProvider.java index bdab4f37715..8c5efa15e44 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/authorize/RMPolicyProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/authorize/RMPolicyProvider.java @@ -18,7 +18,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.security.authorize; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.security.authorize.PolicyProvider; @@ -37,6 +39,23 @@ import org.apache.hadoop.yarn.server.api.ResourceTrackerPB; @InterfaceStability.Unstable public class RMPolicyProvider extends PolicyProvider { + private static RMPolicyProvider rmPolicyProvider = null; + + private RMPolicyProvider() {} + + @Private + @Unstable + public static RMPolicyProvider getInstance() { + if (rmPolicyProvider == null) { + synchronized(RMPolicyProvider.class) { + if (rmPolicyProvider == null) { + rmPolicyProvider = new RMPolicyProvider(); + } + } + } + return rmPolicyProvider; + } + private static final Service[] resourceManagerServices = new Service[] { new Service( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index 31035b420a5..63efe8fe454 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -142,8 +142,15 @@ public class MockRM extends ResourceManager { public void waitForState(MockNM nm, ContainerId containerId, RMContainerState containerState) throws Exception { RMContainer container = getResourceScheduler().getRMContainer(containerId); - Assert.assertNotNull("Container shouldn't be null", container); int timeoutSecs = 0; + while(container == null && timeoutSecs++ < 20) { + nm.nodeHeartbeat(true); + container = getResourceScheduler().getRMContainer(containerId); + System.out.println("Waiting for container " + containerId + " to be allocated."); + Thread.sleep(100); + } + Assert.assertNotNull("Container shouldn't be null", container); + timeoutSecs = 0; while (!containerState.equals(container.getState()) && timeoutSecs++ < 40) { System.out.println("Container : " + containerId + " State is : " + container.getState() + " Waiting for state : " + containerState); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index 5372c18832d..9746664dba8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -24,11 +24,17 @@ import java.io.DataOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.GroupMappingServiceProvider; +import org.apache.hadoop.security.Groups; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.security.authorize.ServiceAuthorizationManager; @@ -38,6 +44,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.junit.After; @@ -105,34 +112,34 @@ public class TestRMAdminService { throws IOException, YarnException { configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); - rm = new MockRM(configuration); - rm.init(configuration); - rm.start(); + try { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + fail("Should throw an exception"); + } catch(Exception ex) { + // Expect exception here + } - // clean the remoteDirectory - cleanRemoteDirectory(); + //upload default configurations + uploadDefaultConfiguration(); + + try { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + } catch(Exception ex) { + fail("Should not get any exceptions"); + } CapacityScheduler cs = (CapacityScheduler) rm.getRMContext().getScheduler(); int maxAppsBefore = cs.getConfiguration().getMaximumSystemApplications(); - try { - rm.adminService.refreshQueues(RefreshQueuesRequest.newInstance()); - fail("FileSystemBasedConfigurationProvider is used." + - " Should get an exception here"); - } catch (Exception ex) { - Assert.assertTrue(ex.getMessage().contains( - "Can not find Configuration: capacity-scheduler.xml")); - } - CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); csConf.set("yarn.scheduler.capacity.maximum-applications", "5000"); - String csConfFile = writeConfigurationXML(csConf, - "capacity-scheduler.xml"); - - // upload the file into Remote File System - uploadToRemoteFileSystem(new Path(csConfFile)); + uploadConfiguration(csConf, "capacity-scheduler.xml"); rm.adminService.refreshQueues(RefreshQueuesRequest.newInstance()); @@ -159,20 +166,24 @@ public class TestRMAdminService { throws IOException, YarnException { configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); - rm = new MockRM(configuration); - rm.init(configuration); - rm.start(); + try { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + fail("Should throw an exception"); + } catch(Exception ex) { + // Expect exception here + } - // clean the remoteDirectory - cleanRemoteDirectory(); + //upload default configurations + uploadDefaultConfiguration(); try { - rm.adminService.refreshAdminAcls(RefreshAdminAclsRequest.newInstance()); - fail("FileSystemBasedConfigurationProvider is used." + - " Should get an exception here"); - } catch (Exception ex) { - Assert.assertTrue(ex.getMessage().contains( - "Can not find Configuration: yarn-site.xml")); + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + } catch(Exception ex) { + fail("Should not get any exceptions"); } String aclStringBefore = @@ -180,10 +191,8 @@ public class TestRMAdminService { YarnConfiguration yarnConf = new YarnConfiguration(); yarnConf.set(YarnConfiguration.YARN_ADMIN_ACL, "world:anyone:rwcda"); - String yarnConfFile = writeConfigurationXML(yarnConf, "yarn-site.xml"); + uploadConfiguration(yarnConf, "yarn-site.xml"); - // upload the file into Remote File System - uploadToRemoteFileSystem(new Path(yarnConfFile)); rm.adminService.refreshAdminAcls(RefreshAdminAclsRequest.newInstance()); String aclStringAfter = @@ -214,7 +223,6 @@ public class TestRMAdminService { } } - @SuppressWarnings("resource") @Test public void testServiceAclsRefreshWithFileSystemBasedConfigurationProvider() throws IOException, YarnException { @@ -224,33 +232,33 @@ public class TestRMAdminService { "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); ResourceManager resourceManager = null; try { - resourceManager = new ResourceManager(); - resourceManager.init(configuration); - resourceManager.start(); - - // clean the remoteDirectory - cleanRemoteDirectory(); - try { - resourceManager.adminService - .refreshServiceAcls(RefreshServiceAclsRequest - .newInstance()); - fail("FileSystemBasedConfigurationProvider is used." + - " Should get an exception here"); + resourceManager = new ResourceManager(); + resourceManager.init(configuration); + resourceManager.start(); + fail("Should throw an exception"); } catch (Exception ex) { - Assert.assertTrue(ex.getMessage().contains( - "Can not find Configuration: hadoop-policy.xml")); + // expect to get an exception here } - String aclsString = "alice,bob users,wheel"; + //upload default configurations + uploadDefaultConfiguration(); Configuration conf = new Configuration(); conf.setBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true); - conf.set("security.applicationclient.protocol.acl", aclsString); - String hadoopConfFile = writeConfigurationXML(conf, "hadoop-policy.xml"); + uploadConfiguration(conf, "core-site.xml"); + try { + resourceManager = new ResourceManager(); + resourceManager.init(configuration); + resourceManager.start(); + } catch (Exception ex) { + fail("Should not get any exceptions"); + } - // upload the file into Remote File System - uploadToRemoteFileSystem(new Path(hadoopConfFile)); + String aclsString = "alice,bob users,wheel"; + Configuration newConf = new Configuration(); + newConf.set("security.applicationclient.protocol.acl", aclsString); + uploadConfiguration(newConf, "hadoop-policy.xml"); resourceManager.adminService.refreshServiceAcls(RefreshServiceAclsRequest .newInstance()); @@ -328,31 +336,31 @@ public class TestRMAdminService { throws IOException, YarnException { configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); - rm = new MockRM(configuration); - rm.init(configuration); - rm.start(); + try { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + fail("Should throw an exception"); + } catch(Exception ex) { + // Expect exception here + } - // clean the remoteDirectory - cleanRemoteDirectory(); + //upload default configurations + uploadDefaultConfiguration(); try { - rm.adminService.refreshSuperUserGroupsConfiguration( - RefreshSuperUserGroupsConfigurationRequest.newInstance()); - fail("FileSystemBasedConfigurationProvider is used." + - " Should get an exception here"); - } catch (Exception ex) { - Assert.assertTrue(ex.getMessage().contains( - "Can not find Configuration: core-site.xml")); + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + } catch(Exception ex) { + fail("Should not get any exceptions"); } Configuration coreConf = new Configuration(false); coreConf.set("hadoop.proxyuser.test.groups", "test_groups"); coreConf.set("hadoop.proxyuser.test.hosts", "test_hosts"); - String coreConfFile = writeConfigurationXML(coreConf, - "core-site.xml"); + uploadConfiguration(coreConf, "core-site.xml"); - // upload the file into Remote File System - uploadToRemoteFileSystem(new Path(coreConfFile)); rm.adminService.refreshSuperUserGroupsConfiguration( RefreshSuperUserGroupsConfigurationRequest.newInstance()); Assert.assertTrue(ProxyUsers.getProxyGroups() @@ -366,6 +374,84 @@ public class TestRMAdminService { .get("hadoop.proxyuser.test.hosts").contains("test_hosts")); } + @Test + public void testRefreshUserToGroupsMappingsWithLocalConfigurationProvider() { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + try { + rm.adminService + .refreshUserToGroupsMappings(RefreshUserToGroupsMappingsRequest + .newInstance()); + } catch (Exception ex) { + fail("Using localConfigurationProvider. Should not get any exception."); + } + } + + @Test + public void + testRefreshUserToGroupsMappingsWithFileSystemBasedConfigurationProvider() + throws IOException, YarnException { + configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); + try { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + fail("Should throw an exception"); + } catch (Exception ex) { + // Expect exception here + } + + String user = UserGroupInformation.getCurrentUser().getUserName(); + List groupWithInit = + new ArrayList(Groups.getUserToGroupsMappingService( + configuration).getGroups(user)); + + // upload default configurations + uploadDefaultConfiguration(); + Configuration conf = new Configuration(); + conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, + MockUnixGroupsMapping.class, + GroupMappingServiceProvider.class); + uploadConfiguration(conf, "core-site.xml"); + + try { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + } catch (Exception ex) { + fail("Should not get any exceptions"); + } + + // Make sure RM will use the updated GroupMappingServiceProvider + List groupBefore = + new ArrayList(Groups.getUserToGroupsMappingService( + configuration).getGroups(user)); + Assert.assertTrue(groupBefore.contains("test_group_A") + && groupBefore.contains("test_group_B") + && groupBefore.contains("test_group_C") && groupBefore.size() == 3); + Assert.assertTrue(groupWithInit.size() != groupBefore.size()); + Assert.assertFalse(groupWithInit.contains("test_group_A") + || groupWithInit.contains("test_group_B") + || groupWithInit.contains("test_group_C")); + + // update the groups + MockUnixGroupsMapping.updateGroups(); + + rm.adminService + .refreshUserToGroupsMappings(RefreshUserToGroupsMappingsRequest + .newInstance()); + List groupAfter = + Groups.getUserToGroupsMappingService(configuration).getGroups(user); + + // should get the updated groups + Assert.assertTrue(groupAfter.contains("test_group_D") + && groupAfter.contains("test_group_E") + && groupAfter.contains("test_group_F") && groupAfter.size() == 3); + + } + private String writeConfigurationXML(Configuration conf, String confXMLName) throws IOException { DataOutputStream output = null; @@ -393,11 +479,63 @@ public class TestRMAdminService { fs.copyFromLocalFile(filePath, workingPath); } - private void cleanRemoteDirectory() throws IOException { - if (fs.exists(workingPath)) { - for (FileStatus file : fs.listStatus(workingPath)) { - fs.delete(file.getPath(), true); - } + private void uploadConfiguration(Configuration conf, String confFileName) + throws IOException { + String csConfFile = writeConfigurationXML(conf, confFileName); + // upload the file into Remote File System + uploadToRemoteFileSystem(new Path(csConfFile)); + } + + private void uploadDefaultConfiguration() throws IOException { + Configuration conf = new Configuration(); + uploadConfiguration(conf, "core-site.xml"); + + YarnConfiguration yarnConf = new YarnConfiguration(); + yarnConf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); + uploadConfiguration(yarnConf, "yarn-site.xml"); + + CapacitySchedulerConfiguration csConf = + new CapacitySchedulerConfiguration(); + uploadConfiguration(csConf, "capacity-scheduler.xml"); + + Configuration hadoopPolicyConf = new Configuration(false); + hadoopPolicyConf + .addResource(YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE); + uploadConfiguration(hadoopPolicyConf, "hadoop-policy.xml"); + } + + private static class MockUnixGroupsMapping implements + GroupMappingServiceProvider { + + @SuppressWarnings("serial") + private static List group = new ArrayList() {{ + add("test_group_A"); + add("test_group_B"); + add("test_group_C"); + }}; + + @Override + public List getGroups(String user) throws IOException { + return group; + } + + @Override + public void cacheGroupsRefresh() throws IOException { + // Do nothing + } + + @Override + public void cacheGroupsAdd(List groups) throws IOException { + // Do nothing + } + + public static void updateGroups() { + group.clear(); + group.add("test_group_D"); + group.add("test_group_E"); + group.add("test_group_F"); } } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java index 41fdca24aef..48fede8c930 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java @@ -23,10 +23,7 @@ import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; import java.util.List; -import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -34,15 +31,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo; import org.apache.hadoop.service.Service; -import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.RMStateVersionPBImpl; @@ -54,6 +44,7 @@ import org.junit.Test; public class TestZKRMStateStore extends RMStateStoreTestBase { public static final Log LOG = LogFactory.getLog(TestZKRMStateStore.class); + private static final int ZK_TIMEOUT_MS = 1000; class TestZKRMStateStoreTester implements RMStateStoreHelper { @@ -141,6 +132,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase { conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true); conf.set(YarnConfiguration.RM_STORE, ZKRMStateStore.class.getName()); conf.set(YarnConfiguration.RM_ZK_ADDRESS, hostPort); + conf.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, ZK_TIMEOUT_MS); conf.set(YarnConfiguration.RM_HA_ID, rmId); for (String rpcAddress : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { for (String id : HAUtil.getRMHAIds(conf)) { @@ -182,26 +174,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase { HAServiceProtocol.HAServiceState.ACTIVE, rm2.getRMContext().getRMAdminService().getServiceStatus().getState()); - // Submitting an application to RM1 to trigger a state store operation. - // RM1 should realize that it got fenced and is not the Active RM anymore. - Map mockMap = mock(Map.class); - ApplicationSubmissionContext asc = - ApplicationSubmissionContext.newInstance( - ApplicationId.newInstance(1000, 1), - "testApplication", // app Name - "default", // queue name - Priority.newInstance(0), - ContainerLaunchContext.newInstance(mockMap, mockMap, - new ArrayList(), mockMap, mock(ByteBuffer.class), - mockMap), - false, // unmanaged AM - true, // cancelTokens - 1, // max app attempts - Resource.newInstance(1024, 1)); - ClientRMService rmService = rm1.getClientRMService(); - rmService.submitApplication(SubmitApplicationRequest.newInstance(asc)); - - for (int i = 0; i < 30; i++) { + for (int i = 0; i < ZK_TIMEOUT_MS / 50; i++) { if (HAServiceProtocol.HAServiceState.ACTIVE == rm1.getRMContext().getRMAdminService().getServiceStatus().getState()) { Thread.sleep(100); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index 954a4845c32..f9550f049ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.event.InlineDispatcher; @@ -695,6 +696,8 @@ public class TestRMAppAttemptTransitions { // launch AM and verify attempt failed applicationAttempt.handle(new RMAppAttemptRegistrationEvent( applicationAttempt.getAppAttemptId(), "host", 8042, "oldtrackingurl")); + assertEquals(YarnApplicationAttemptState.SUBMITTED, + applicationAttempt.createApplicationAttemptState()); testAppAttemptSubmittedToFailedState( "Unmanaged AM must register after AM attempt reaches LAUNCHED state."); } @@ -710,6 +713,8 @@ public class TestRMAppAttemptTransitions { // launch AM and verify attempt failed applicationAttempt.handle(new RMAppAttemptRegistrationEvent( applicationAttempt.getAppAttemptId(), "host", 8042, "oldtrackingurl")); + assertEquals(YarnApplicationAttemptState.SUBMITTED, + applicationAttempt.createApplicationAttemptState()); sendAttemptUpdateSavedEvent(applicationAttempt); assertFalse(transferStateFromPreviousAttempt); } @@ -720,6 +725,8 @@ public class TestRMAppAttemptTransitions { new RMAppAttemptEvent( applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.KILL)); + assertEquals(YarnApplicationAttemptState.NEW, + applicationAttempt.createApplicationAttemptState()); testAppAttemptKilledState(null, EMPTY_DIAGNOSTICS); verifyTokenCount(applicationAttempt.getAppAttemptId(), 1); } @@ -740,6 +747,8 @@ public class TestRMAppAttemptTransitions { new RMAppAttemptEvent( applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.KILL)); + assertEquals(YarnApplicationAttemptState.SUBMITTED, + applicationAttempt.createApplicationAttemptState()); testAppAttemptKilledState(null, EMPTY_DIAGNOSTICS); } @@ -750,6 +759,8 @@ public class TestRMAppAttemptTransitions { new RMAppAttemptEvent( applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.KILL)); + assertEquals(YarnApplicationAttemptState.SCHEDULED, + applicationAttempt.createApplicationAttemptState()); testAppAttemptKilledState(null, EMPTY_DIAGNOSTICS); } @@ -760,6 +771,8 @@ public class TestRMAppAttemptTransitions { new RMAppAttemptEvent( applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.KILL)); + assertEquals(YarnApplicationAttemptState.ALLOCATED, + applicationAttempt.createApplicationAttemptState()); testAppAttemptKilledState(amContainer, EMPTY_DIAGNOSTICS); } @@ -771,6 +784,8 @@ public class TestRMAppAttemptTransitions { new RMAppAttemptLaunchFailedEvent( applicationAttempt.getAppAttemptId(), diagnostics)); + assertEquals(YarnApplicationAttemptState.ALLOCATED, + applicationAttempt.createApplicationAttemptState()); testAppAttemptFailedState(amContainer, diagnostics); } @@ -784,6 +799,8 @@ public class TestRMAppAttemptTransitions { ContainerState.COMPLETE, containerDiagMsg, exitCode); applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent( applicationAttempt.getAppAttemptId(), cs)); + assertEquals(YarnApplicationAttemptState.ALLOCATED, + applicationAttempt.createApplicationAttemptState()); sendAttemptUpdateSavedEvent(applicationAttempt); assertEquals(RMAppAttemptState.FAILED, applicationAttempt.getAppAttemptState()); @@ -815,7 +832,8 @@ public class TestRMAppAttemptTransitions { applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.EXPIRE)); assertEquals(RMAppAttemptState.FINAL_SAVING, applicationAttempt.getAppAttemptState()); - + assertEquals(YarnApplicationAttemptState.RUNNING, + applicationAttempt.createApplicationAttemptState()); sendAttemptUpdateSavedEvent(applicationAttempt); assertEquals(RMAppAttemptState.FAILED, applicationAttempt.getAppAttemptState()); @@ -850,7 +868,8 @@ public class TestRMAppAttemptTransitions { applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.EXPIRE)); assertEquals(RMAppAttemptState.FINAL_SAVING, applicationAttempt.getAppAttemptState()); - + assertEquals(YarnApplicationAttemptState.RUNNING, + applicationAttempt.createApplicationAttemptState()); sendAttemptUpdateSavedEvent(applicationAttempt); assertEquals(RMAppAttemptState.KILLED, applicationAttempt.getAppAttemptState()); @@ -871,6 +890,8 @@ public class TestRMAppAttemptTransitions { launchApplicationAttempt(amContainer); applicationAttempt.handle(new RMAppAttemptEvent( applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.EXPIRE)); + assertEquals(YarnApplicationAttemptState.LAUNCHED, + applicationAttempt.createApplicationAttemptState()); sendAttemptUpdateSavedEvent(applicationAttempt); assertEquals(RMAppAttemptState.FAILED, applicationAttempt.getAppAttemptState()); @@ -890,6 +911,8 @@ public class TestRMAppAttemptTransitions { runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false); applicationAttempt.handle(new RMAppAttemptEvent( applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.EXPIRE)); + assertEquals(YarnApplicationAttemptState.RUNNING, + applicationAttempt.createApplicationAttemptState()); sendAttemptUpdateSavedEvent(applicationAttempt); assertEquals(RMAppAttemptState.FAILED, applicationAttempt.getAppAttemptState()); @@ -1055,6 +1078,8 @@ public class TestRMAppAttemptTransitions { diagnostics)); assertEquals(RMAppAttemptState.FINAL_SAVING, applicationAttempt.getAppAttemptState()); + assertEquals(YarnApplicationAttemptState.RUNNING, + applicationAttempt.createApplicationAttemptState()); // Container_finished event comes before Attempt_Saved event. applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent( applicationAttempt.getAppAttemptId(), BuilderUtils.newContainerStatus( @@ -1083,6 +1108,8 @@ public class TestRMAppAttemptTransitions { diagnostics)); assertEquals(RMAppAttemptState.FINAL_SAVING, applicationAttempt.getAppAttemptState()); + assertEquals(YarnApplicationAttemptState.RUNNING, + applicationAttempt.createApplicationAttemptState()); // Expire event comes before Attempt_saved event. applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt .getAppAttemptId(), RMAppAttemptEventType.EXPIRE)); @@ -1118,6 +1145,8 @@ public class TestRMAppAttemptTransitions { applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt .getAppAttemptId(), RMAppAttemptEventType.KILL)); + assertEquals(YarnApplicationAttemptState.LAUNCHED, + applicationAttempt.createApplicationAttemptState()); sendAttemptUpdateSavedEvent(applicationAttempt); // after attempt is killed, can not get Client Token token = applicationAttempt.createClientToken(null); @@ -1140,6 +1169,8 @@ public class TestRMAppAttemptTransitions { ApplicationAttemptId appAttemptId = applicationAttempt.getAppAttemptId(); applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent( appAttemptId, cs1)); + assertEquals(YarnApplicationAttemptState.RUNNING, + applicationAttempt.createApplicationAttemptState()); sendAttemptUpdateSavedEvent(applicationAttempt); assertEquals(RMAppAttemptState.FAILED, applicationAttempt.getAppAttemptState()); @@ -1178,6 +1209,8 @@ public class TestRMAppAttemptTransitions { ApplicationAttemptId appAttemptId = applicationAttempt.getAppAttemptId(); applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent( appAttemptId, cs1)); + assertEquals(YarnApplicationAttemptState.RUNNING, + applicationAttempt.createApplicationAttemptState()); sendAttemptUpdateSavedEvent(applicationAttempt); assertEquals(RMAppAttemptState.FAILED, applicationAttempt.getAppAttemptState()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index ca60db3f04c..47ec5462350 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -40,6 +40,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.net.NetworkTopology; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -104,6 +105,7 @@ public class TestCapacityScheduler { private static float B3_CAPACITY = 20; private ResourceManager resourceManager = null; + private RMContext mockContext; @Before public void setUp() throws Exception { @@ -118,6 +120,9 @@ public class TestCapacityScheduler { resourceManager.getRMContainerTokenSecretManager().rollMasterKey(); resourceManager.getRMNMTokenSecretManager().rollMasterKey(); ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start(); + mockContext = mock(RMContext.class); + when(mockContext.getConfigurationProvider()).thenReturn( + new LocalConfigurationProvider()); } @After @@ -133,7 +138,7 @@ public class TestCapacityScheduler { conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 2048); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024); try { - scheduler.reinitialize(conf, null); + scheduler.reinitialize(conf, mockContext); fail("Exception is expected because the min memory allocation is" + " larger than the max memory allocation."); } catch (YarnRuntimeException e) { @@ -147,7 +152,7 @@ public class TestCapacityScheduler { conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 2); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, 1); try { - scheduler.reinitialize(conf, null); + scheduler.reinitialize(conf, mockContext); fail("Exception is expected because the min vcores allocation is" + " larger than the max vcores allocation."); } catch (YarnRuntimeException e) { @@ -353,7 +358,7 @@ public class TestCapacityScheduler { conf.setCapacity(A, 80f); conf.setCapacity(B, 20f); - cs.reinitialize(conf,null); + cs.reinitialize(conf, mockContext); checkQueueCapacities(cs, 80f, 20f); } @@ -503,7 +508,7 @@ public class TestCapacityScheduler { conf.setCapacity(B2, B2_CAPACITY); conf.setCapacity(B3, B3_CAPACITY); conf.setCapacity(B4, B4_CAPACITY); - cs.reinitialize(conf,null); + cs.reinitialize(conf,mockContext); checkQueueCapacities(cs, 80f, 20f); // Verify parent for B4 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java index b877fbbf98f..0e3bdeb2d4a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java @@ -18,11 +18,17 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; +import java.util.ArrayList; +import java.util.List; + import junit.framework.Assert; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; @@ -30,6 +36,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.TestFifoScheduler; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.junit.Test; @@ -106,4 +115,38 @@ public class TestContainerAllocation { rm.stop(); } + + // This is to test container tokens are generated when the containers are + // acquired by the AM, not when the containers are allocated + @Test + public void testContainerTokenGeneratedOnPullRequest() throws Exception { + YarnConfiguration conf = new YarnConfiguration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + MockRM rm1 = new MockRM(conf); + rm1.start(); + MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 8000); + RMApp app1 = rm1.submitApp(200); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); + // request a container. + am1.allocate("127.0.0.1", 1024, 1, new ArrayList()); + ContainerId containerId2 = + ContainerId.newInstance(am1.getApplicationAttemptId(), 2); + rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED); + + RMContainer container = + rm1.getResourceScheduler().getRMContainer(containerId2); + // no container token is generated. + Assert.assertEquals(containerId2, container.getContainerId()); + Assert.assertNull(container.getContainer().getContainerToken()); + + // acquire the container. + List containers = + am1.allocate(new ArrayList(), + new ArrayList()).getAllocatedContainers(); + Assert.assertEquals(containerId2, containers.get(0).getId()); + // container token is generated. + Assert.assertNotNull(containers.get(0).getContainerToken()); + rm1.stop(); + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm index 386be09d483..a0a9e7412af 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm @@ -35,7 +35,7 @@ Usage: yarn [--config confdir] COMMAND *---------------+--------------+ || COMMAND_OPTIONS || Description | *---------------+--------------+ -| --config confdir | Overwrites the default Configuration directory. Default is ${HADOOP_PREFIX}/conf. | +| --config confdir | Overwrites the default Configuration directory. Default is $\{HADOOP_PREFIX\}/conf. | *---------------+--------------+ | COMMAND COMMAND_OPTIONS | Various commands with their options are described in the following sections. The commands have been grouped into {{User Commands}} and {{Administration Commands}}. | *---------------+--------------+ @@ -63,11 +63,22 @@ Usage: yarn [--config confdir] COMMAND *---------------+--------------+ || COMMAND_OPTIONS || Description | *---------------+--------------+ -| -status ApplicationId | Specify an application id | +| -list | Lists applications from the RM. Supports optional use of -appTypes +| | to filter applications based on application type, and -appStates to +| | filter applications based on application state. *---------------+--------------+ -| -list | Lists all the Applications from RM | +| -appStates States | Works with -list to filter applications based on input +| | comma-separated list of application states. The valid +| | application state can be one of the following: \ +| | ALL, NEW, NEW_SAVING, SUBMITTED, ACCEPTED, RUNNING, +| | FINISHED, FAILED, KILLED *---------------+--------------+ -| -kill ApplicationId | Specify an application id | +| -appTypes Types | Works with -list to filter applications based on input +| | comma-separated list of application types. +*---------------+--------------+ +| -status ApplicationId | Prints the status of the application. +*---------------+--------------+ +| -kill ApplicationId | Kills the application. *---------------+--------------+ ** node @@ -81,9 +92,15 @@ Usage: yarn [--config confdir] COMMAND *---------------+--------------+ || COMMAND_OPTIONS || Description | *---------------+--------------+ -| -status NodeId | Specify a node id | +| -list | Lists all running nodes. Supports optional use of -states to filter +| | nodes based on node state, and -all to list all nodes. *---------------+--------------+ -| -list | Lists all the Nodes | +| -states States | Works with -list to filter nodes based on input +| | comma-separated list of node states. +*---------------+--------------+ +| -all | Works with -list to list all nodes. +*---------------+--------------+ +| -status NodeId | Prints the status report of the node. *---------------+--------------+ ** logs @@ -91,19 +108,22 @@ Usage: yarn [--config confdir] COMMAND Dump the container logs ------- - Usage: yarn logs + Usage: yarn logs -applicationId ------- *---------------+--------------+ || COMMAND_OPTIONS || Description | *---------------+--------------+ -| -applicationId ApplicationId | Specify an application id | +| -applicationId \ | Specifies an application id | *---------------+--------------+ -| -appOwner AppOwner | Specify an application owner | +| -appOwner AppOwner | AppOwner (assumed to be current user if not +| | specified) *---------------+--------------+ -| -containerId ContainerId | Specify a container id | +| -containerId ContainerId | ContainerId (must be specified if node address is +| | specified) *---------------+--------------+ -| -nodeAddress NodeAddress | Specify a node address | +| -nodeAddress NodeAddress | NodeAddress in the format nodename:port (must be +| | specified if container id is specified) *---------------+--------------+ ** classpath @@ -158,7 +178,11 @@ Usage: yarn [--config confdir] COMMAND ------- Usage: yarn rmadmin [-refreshQueues] [-refreshNodes] [-refreshUserToGroupsMapping] [-refreshSuperUserGroupsConfiguration] [-refreshAdminAcls] - [-refreshServiceAcl] [-help [cmd]] + [-refreshServiceAcl] [-getGroups [username]] [-help [cmd]] + [-transitionToActive ] + [-transitionToStandby ] + [-getServiceState ] + [-checkHealth ] ------- *---------------+--------------+ @@ -176,8 +200,22 @@ Usage: yarn [--config confdir] COMMAND *---------------+--------------+ | -refreshServiceAcl | Reload the service-level authorization policy file ResoureceManager will reload the authorization policy file. | *---------------+--------------+ +| -getGroups [username] | Get groups the specified user belongs to. +*---------------+--------------+ | -help [cmd] | Displays help for the given command or all commands if none is specified. | *---------------+--------------+ +| -transitionToActive \ | Transitions the service into Active +| | state. +*---------------+--------------+ +| -transitionToStandby \ | Transitions the service into Standby +| | state. +*---------------+--------------+ +| -getServiceState \ | Returns the state of the service. +*---------------+--------------+ +| -checkHealth \ | Requests that the service perform a health +| | check. The RMAdmin tool will exit with a +| | non-zero exit code if the check fails. +*---------------+--------------+ ** daemonlog @@ -191,9 +229,9 @@ Usage: yarn [--config confdir] COMMAND *---------------+--------------+ || COMMAND_OPTIONS || Description | *---------------+--------------+ -| -getlevel | Prints the log level of the daemon running at . This command internally connects to http:///logLevel?log= | +| -getlevel \ \ | Prints the log level of the daemon running at \. This command internally connects to http://\/logLevel?log=\ *---------------+--------------+ -| -setlevel | Sets the log level of the daemon running at . This command internally connects to http:///logLevel?log= | +| -setlevel \ \ \ | Sets the log level of the daemon running at \. This command internally connects to http://\/logLevel?log=\ *---------------+--------------+